diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml new file mode 100644 index 00000000..7e888b67 --- /dev/null +++ b/.github/actions/debusine-build/action.yml @@ -0,0 +1,131 @@ +name: Build in Debusine +description: > + Submit the source package the prepare job built to Debusine, returning the + CI workspace it was built in. + +# Called by build-kernel-debian.yml, which decides what becomes of the result: +# every build is downloaded and uploaded to S3, and one whose caller named a +# target workspace is also promoted into it. Holding these steps in a composite +# action keeps the submission itself in one place, separate from the job that +# declares the runner, container and environment it happens in. +# +# The caller must have checked out pkg-linux-qcom into the workspace first, +# which is also where this action comes from. Runs inside the +# debusine-pkg-builder container. +# +# vars and secrets contexts do not reach a composite action, so the Debusine +# host, scope and credentials come in as inputs from the calling workflow. + +inputs: + suite: + description: 'Target Debian suite (trixie, forky, sid)' + required: true + flavour: + description: 'Kernel flavour, isolating artifacts and Debusine workspaces' + required: true + debusine-host: + description: 'Debusine instance hostname' + required: true + debusine-scope: + description: 'Debusine scope' + required: true + debusine-user: + description: 'Debusine account used to submit the build' + required: true + debusine-token: + description: 'Debusine token for the CI child workspace' + required: true + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + default: 'qli-ci' + extra-build-dep-workspaces: + description: > + Workspaces the build resolves its build-dependencies from, space + separated. Set by the calling workflow, which knows whether this build is + one a release may promote. + default: 'qli' + debusine-action-ref: + description: 'Ref of qualcomm-linux/debusine-action to use' + default: 'main' + +outputs: + workspace: + description: 'Debusine CI workspace the package was built in' + value: ${{ steps.build-debusine.outputs.workspace }} + workspace_url: + description: 'Browser URL of that workspace' + value: ${{ steps.build-debusine.outputs.workspace_url }} + srcpkg_version: + description: 'Version of the submitted source package' + value: ${{ steps.srcpkg.outputs.srcpkg_version }} + +runs: + using: composite + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: ${{ inputs.debusine-action-ref }} + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + # Into the workspace root: lib/build takes the .dsc from its working + # directory, and reaches its own helpers by a path relative to it, so the + # two have to be the same directory. + - name: Download source package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} + + - name: Read the source package version + # From the .dsc as downloaded, which is what Debusine will be given; + # the prepare job reported the same value from the same file. + id: srcpkg + shell: bash + run: | + set -euo pipefail + DSC=$(ls -- *.dsc) + [[ $(wc -l <<<"$DSC") -eq 1 ]] || { + echo "::error::Expected exactly one .dsc in the source package artifact, found: $DSC" + exit 1 + } + SRCPKG_VERSION=$(sed -n 's/^Version: //p' "$DSC") + [[ "$SRCPKG_VERSION" =~ ^[A-Za-z0-9.+~:-]+$ ]] || { + echo "::error::Could not read a version from $DSC (got '$SRCPKG_VERSION')" + exit 1 + } + echo "Submitting $DSC ($SRCPKG_VERSION)" + echo "srcpkg_version=$SRCPKG_VERSION" >> "$GITHUB_OUTPUT" + + - name: Build in Debusine + id: build-debusine + shell: bash + env: + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + JOB_INDEX: ${{ inputs.flavour }}-${{ inputs.suite }} + DEBUSINE_HOST: ${{ inputs.debusine-host }} + DEBUSINE_SCOPE: ${{ inputs.debusine-scope }} + DEBUSINE_USER: ${{ inputs.debusine-user }} + DEBUSINE_TOKEN: ${{ inputs.debusine-token }} + DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} + SUITE: ${{ inputs.suite }} + EXTRA_BUILD_DEP_WORKSPACES: ${{ inputs.extra-build-dep-workspaces }} + run: | + # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. + set -euo pipefail + debusine-action/lib/build + workspace=$(sed -n 's/^workspace=//p' "$GITHUB_OUTPUT") + echo "workspace_url=https://${DEBUSINE_HOST}/${DEBUSINE_SCOPE}/${workspace}/" >> "$GITHUB_OUTPUT" + + - name: Note Debusine workspace URL + shell: bash + env: + WORKSPACE_URL: ${{ steps.build-debusine.outputs.workspace_url }} + run: | + echo "Debusine Workspace URL: $WORKSPACE_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml new file mode 100644 index 00000000..fe6dd19d --- /dev/null +++ b/.github/actions/prepare-kernel-source/action.yml @@ -0,0 +1,430 @@ +name: Prepare kernel source +description: > + Resolve the kernel ref, clone it, inject debian/, build the Debian source + package, and upload it as an artifact for the build job to consume. + +# Every build takes this path, whichever family it belongs to, so it lives in a +# composite action rather than a reusable workflow: the two family workflows +# call it as a step of their own prepare job, which keeps one copy of these +# steps without adding a nesting level to every job name in the checks list. +# +# The caller must have checked out pkg-linux-qcom into the workspace first: +# debian/ and ci/ come from that checkout, and so does this action. +# +# Runs on the self-hosted arm64 runner, and leaves nothing behind in the job +# environment: values pass between these steps as step outputs, each consumer +# naming what it reads. Writing them to GITHUB_ENV instead would put them in +# the environment of every later step in the job, including steps outside this +# action, and a newline in one would let the value that carried it define +# environment variables of its own. Several of these values are not ours: the +# kernel ref comes from a remote's tag list, and on pr-build.yml the inputs +# come from the build matrix as the pull request wrote it. + +outputs: + srcpkg_name: + description: 'Name of the source package this build produces' + value: ${{ steps.version.outputs.srcpkg_name }} + srcpkg_version: + description: > + Version of the source package this build produces, read from the .dsc + build-source-package.sh wrote. A caller that promotes the result into + an archive needs it before the build starts, to see whether that + version is already published there. + value: ${{ steps.version.outputs.srcpkg_version }} + orig_file: + description: 'Filename of the .orig.tar.gz the source package references' + value: ${{ steps.version.outputs.orig_file }} + orig_sha256: + description: > + SHA-256 of that orig tarball. It is a function of the kernel commit, so + a caller can compare it with what an archive already holds for this + upstream version and fail on a difference. + value: ${{ steps.version.outputs.orig_sha256 }} + +inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + required: true + suite: + description: 'Target suite' + required: true + registry-token: + description: > + Token used to pull the pkg-builder image from ghcr.io, which is not a + public package. Defaults to the job's own GITHUB_TOKEN, which needs the + packages: read permission the calling workflows declare. The secrets + context does not reach a composite action, hence an input. + default: ${{ github.token }} + expect-family: + description: > + Build family the calling workflow builds, debian or ubuntu. The suite is + a free-text matrix value, so this catches one handed to the workflow for + the other family before anything is cloned or built. + required: true + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + required: true + kernel-url: + description: 'Kernel repository URL' + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + default: 'latest_tag' + kernel-branch: + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' + default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag' + default: 'qcom-next-*' + srcpkg: + description: 'Debian source package name' + default: 'linux-qcom-next' + binpkg: + description: 'Kernel image metapackage name' + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + default: '' + debian-revision: + description: 'Debian revision component of the package version, from the matrix entry' + required: true + localversion: + description: 'LOCALVERSION override (auto-derived from the resolved ref if empty)' + default: '' + kver-extra: + description: 'Extra package-version suffix (e.g. -ci42)' + default: '' + +runs: + using: composite + steps: + - name: Resolve build environment + id: env + shell: bash + env: + SUITE_INPUT: ${{ inputs.suite }} + EXPECT_FAMILY: ${{ inputs.expect-family }} + run: | + set -euo pipefail + # Family and docker image in one place, because they answer the same + # question about a suite. The family list here is the one in + # ci/scripts/resolve-matrix.py, which routes an entry to the workflow + # that calls this action. + # + # Ubuntu-family suites use an image of their own suite. Every + # Debian-family suite uses the trixie image, forky and sid included: + # the registry publishes no image for either, and docker-pkg-build has + # no Dockerfile to build one from, so they borrow trixie's until one + # exists. That costs nothing today because Debusine performs the actual + # suite-specific build; this container only has to run + # prepare-source.sh, which generates packaging rather than compiling + # anything. + case "$SUITE_INPUT" in + trixie|forky|sid|unstable) FAMILY=debian; DOCKER_DISTRO=trixie ;; + *) FAMILY=ubuntu; DOCKER_DISTRO="$SUITE_INPUT" ;; + esac + + if [[ "$FAMILY" != "$EXPECT_FAMILY" ]]; then + echo "::error::Suite $SUITE_INPUT is $FAMILY-family, but this workflow builds the $EXPECT_FAMILY family." + echo "::error::Dispatch the $FAMILY build workflow for it instead." + exit 1 + fi + + # Pull the images qualcomm-linux/docker-pkg-build publishes, and build + # the rest here. Its container-build-and-upload.yml pushes trixie and + # resolute (and noble) on every push to main and weekly, having + # validated each with a test package build, so rebuilding them from the + # same Dockerfile on every job repeats work already done. Anything it + # does not publish still has to be built on the runner. + case "$DOCKER_DISTRO" in + trixie|resolute) IMAGE_SOURCE=pull ;; + *) IMAGE_SOURCE=build ;; + esac + + echo "distro=$SUITE_INPUT" >> "$GITHUB_OUTPUT" + echo "docker_distro=$DOCKER_DISTRO" >> "$GITHUB_OUTPUT" + echo "image_source=$IMAGE_SOURCE" >> "$GITHUB_OUTPUT" + echo "Suite $SUITE_INPUT is $FAMILY-family and uses the $DOCKER_DISTRO image ($IMAGE_SOURCE)" + + - name: Pull builder image + if: ${{ steps.env.outputs.image_source == 'pull' }} + shell: bash + env: + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + REGISTRY_TOKEN: ${{ inputs.registry-token }} + run: | + # No -x: REGISTRY_TOKEN is in env. + set -euo pipefail + # ghcr.io/qualcomm-linux/pkg-builder is not a public package, so the + # pull needs a login even though the image is only ever read. + echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + docker pull "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" + docker image ls + + # Only for a suite with no published image: the pull path needs neither + # this checkout nor the build below. + - name: Checkout docker-pkg-build + if: ${{ steps.env.outputs.image_source == 'build' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/docker-pkg-build + ref: main + path: docker-pkg-build + + - name: Build builder image + if: ${{ steps.env.outputs.image_source == 'build' }} + shell: bash + env: + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + run: | + set -euo pipefail + ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" + docker image ls + + - name: Resolve kernel ref + id: ref + shell: bash + env: + KERNEL_URL_INPUT: ${{ inputs.kernel-url }} + BRANCH_INPUT: ${{ inputs.kernel-branch }} + REF_STRATEGY_INPUT: ${{ inputs.ref-strategy }} + TAG_PATTERN_INPUT: ${{ inputs.tag-pattern }} + BUILD_INPUT: ${{ inputs.build }} + run: | + set -euo pipefail + [[ "$BUILD_INPUT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "::error::build name must use lowercase letters, digits, and internal hyphens" + exit 1 + } + case "$REF_STRATEGY_INPUT" in + latest_tag) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ + --url "$KERNEL_URL_INPUT" \ + --latest-tag "$TAG_PATTERN_INPUT") + ;; + branch_tip|pinned_ref) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ + --url "$KERNEL_URL_INPUT" --ref "$BRANCH_INPUT") + ;; + *) + echo "::error::unsupported ref strategy: $REF_STRATEGY_INPUT" + exit 1 + ;; + esac + + # A ref carrying a newline would otherwise define outputs of its own. + # The remote chooses this value, so check it before writing it out. + [[ "$KERNEL_REF" == *$'\n'* ]] && { + echo "::error::resolved kernel ref contains a newline" + exit 1 + } + + echo "kernel_ref=$KERNEL_REF" >> "$GITHUB_OUTPUT" + echo "Build: $BUILD_INPUT" + echo "Ref strategy: $REF_STRATEGY_INPUT" + echo "Resolved kernel ref: $KERNEL_REF" + + - name: Clone kernel source + id: clone + shell: bash + env: + KERNEL_URL: ${{ inputs.kernel-url }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + run: | + set -euo pipefail + KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" + echo "kernel_dir=$KERNEL_DIR" >> "$GITHUB_OUTPUT" + echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." + git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ + "$KERNEL_URL" "$KERNEL_DIR" + KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) + echo "kernel_sha=$KERNEL_SHA" >> "$GITHUB_OUTPUT" + echo "Kernel HEAD SHA: $KERNEL_SHA" + + # Committer date of HEAD, normalised to UTC. Committer rather than + # author date: an author date can be months old on a backported patch, + # which would date the build by when someone first wrote the code + # rather than by when this snapshot came to exist. + KERNEL_DATE=$(TZ=UTC git -C "$KERNEL_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) + [[ "$KERNEL_DATE" =~ ^[0-9]{8}$ ]] || { + echo "::error::could not read a YYYYMMDD commit date from $KERNEL_REF (got '$KERNEL_DATE')" + exit 1 + } + echo "kernel_date=$KERNEL_DATE" >> "$GITHUB_OUTPUT" + echo "Kernel HEAD commit date (UTC): $KERNEL_DATE" + + # The same instant in RFC 2822 form, for the changelog entry. Read + # here, on the host that owns the clone, because prepare-source.sh + # runs in a container as root and git will not read a checkout owned + # by someone else. + CHANGELOG_DATE=$(git -C "$KERNEL_DIR" log -1 --format=%cD) + echo "changelog_date=$CHANGELOG_DATE" >> "$GITHUB_OUTPUT" + + - name: Derive version fields + id: localversion + shell: bash + env: + LOCALVERSION_INPUT: ${{ inputs.localversion }} + FLAVOUR_INPUT: ${{ inputs.flavour }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} + KERNEL_DATE: ${{ steps.clone.outputs.kernel_date }} + run: | + set -euo pipefail + if [[ -n "$LOCALVERSION_INPUT" ]]; then + # An override supplies no snapshot, and the ref it was chosen to + # replace cannot be trusted to describe it. The Debian version then + # carries no +git; prepare-source.sh warns about the same. + echo "Using explicit LOCALVERSION override: $LOCALVERSION_INPUT" + echo "localversion=$LOCALVERSION_INPUT" >> "$GITHUB_OUTPUT" + echo "snapshot=" >> "$GITHUB_OUTPUT" + echo "gitsha=" >> "$GITHUB_OUTPUT" + else + # Emits LOCALVERSION=, SNAPSHOT= and GITSHA= lines. They are read + # here rather than recovered from LOCALVERSION later: that string + # also carries a flavour name, and a hex SHA can end in eight digits + # of its own. + # + # --date is only consulted for branch-tip builds, where the ref + # carries no snapshot. It comes from the commit rather than the + # clock, so rebuilding a commit reproduces its version instead of + # inventing a higher one. + FIELDS=$(ci/scripts/derive-localversion.sh \ + --flavour "$FLAVOUR_INPUT" \ + --ref "$KERNEL_REF" \ + --sha "$KERNEL_SHA" \ + --date "$KERNEL_DATE") + echo "Derived version fields:" + echo "$FIELDS" + echo "$FIELDS" | while IFS='=' read -r key value; do + case "$key" in + LOCALVERSION) echo "localversion=$value" ;; + SNAPSHOT) echo "snapshot=$value" ;; + GITSHA) echo "gitsha=$value" ;; + *) echo "::error::unexpected field '$key' from derive-localversion.sh"; exit 1 ;; + esac + done >> "$GITHUB_OUTPUT" + fi + + - name: Prepare source + shell: bash + env: + SRCPKG_INPUT: ${{ inputs.srcpkg }} + BINPKG_INPUT: ${{ inputs.binpkg }} + KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} + KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} + DKMS_INPUT: ${{ inputs.dkms }} + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + DISTRO: ${{ steps.env.outputs.distro }} + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + LOCALVERSION: ${{ steps.localversion.outputs.localversion }} + SNAPSHOT: ${{ steps.localversion.outputs.snapshot }} + DEBIAN_REVISION: ${{ inputs.debian-revision }} + KERNEL_URL: ${{ inputs.kernel-url }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} + CHANGELOG_DATE: ${{ steps.clone.outputs.changelog_date }} + run: | + # No -x here: keep xtrace off for this step as it assembles the + # prepare-source.sh argument list which may include sensitive paths. + set -euo pipefail + + ARGS=( + --source-dir "$KERNEL_DIR" + --distro "$DISTRO" + --srcpkg "$SRCPKG_INPUT" + --binpkg "$BINPKG_INPUT" + --debian-revision "$DEBIAN_REVISION" + --changelog-date "$CHANGELOG_DATE" + ) + + # The version fields all come from derive-localversion.sh, which + # composed them from the ref and HEAD together. Nothing here appends to + # them: the SHA is already inside LOCALVERSION, and SNAPSHOT and the + # SHA are passed separately for the Debian version, which joins them + # differently. + [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") + [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") + # Full SHA: prepare-source.sh takes the first 12 for the version + # strings and records the whole thing in the changelog, alongside the + # repository and ref that the version strings do not name. + ARGS+=(--git-sha "$KERNEL_SHA" --git-clone "$KERNEL_URL" --git-ref "$KERNEL_REF") + [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") + [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") + # Always passed, unlike the optional inputs above: an empty list is a + # genuine "bundle nothing" rather than a request for some default. + ARGS+=(--dkms "$DKMS_INPUT") + + docker run -i --rm \ + -v "$PWD:$PWD" --workdir="$PWD" \ + "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ + ./prepare-source.sh "${ARGS[@]}" + + - name: Build source package + # The .orig.tar.gz, .dsc and .changes the binary build starts from, + # whichever family builds it. Built here, in the same container that + # ran prepare-source.sh, because the runner host has no dpkg-dev and + # because the orig's bytes depend on the gzip that wrote them: every + # build that must agree on a checksum writes its orig in this image. + # + # git is told to trust the checkout, which the runner owns and the + # container reads as root; the checkout is this job's own clone. The + # fields file lands beside the output directory, outside the artifact. + shell: bash + env: + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + run: | + set -euo pipefail + docker run -i --rm \ + -v "$PWD:$PWD" --workdir="$PWD" \ + -e GIT_CONFIG_COUNT=1 \ + -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0='*' \ + "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ + ./build-source-package.sh \ + --source-dir "$KERNEL_DIR" \ + --output-dir "$PWD/source-package" \ + --write-fields "$PWD/source-package.fields" + + - name: Read the source package fields + # build-source-package.sh has decided the version and written it into + # the .dsc, so what it reports is what will be built and nothing can + # disagree with it. Read from its fields file rather than recomputed: + # these steps run on the runner host, which has no dpkg-dev. The values + # are the script's own, filtered to the characters Debian allows in a + # name and version, so none can carry a newline into GITHUB_OUTPUT. + id: version + shell: bash + run: | + set -euo pipefail + while IFS='=' read -r key value; do + case "$key" in + SRCPKG_NAME) echo "srcpkg_name=$value" ;; + SRCPKG_VERSION) echo "srcpkg_version=$value" ;; + ORIG) echo "orig_file=$(basename "$value")" ;; + ORIG_SHA256) echo "orig_sha256=$value" ;; + UPSTREAM_VERSION|CHANGES|DSC) ;; + *) echo "::error::unexpected field '$key' from build-source-package.sh"; exit 1 ;; + esac + done < source-package.fields >> "$GITHUB_OUTPUT" + grep -q '^SRCPKG_VERSION=.' source-package.fields || { + echo "::error::build-source-package.sh reported no version." + exit 1 + } + echo "Source package:" + cat source-package.fields + + - name: Upload source package artifact + # The whole .changes set: .changes, .dsc, .debian.tar.xz and the orig. + # Plain files with no execute bits to lose, so no tarball around them. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} + path: source-package/ + if-no-files-found: error + retention-days: 7 + diff --git a/.github/dependabots.yaml b/.github/dependabot.yml similarity index 100% rename from .github/dependabots.yaml rename to .github/dependabot.yml diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml deleted file mode 100644 index 9da42580..00000000 --- a/.github/workflows/build-kernel-deb.yml +++ /dev/null @@ -1,460 +0,0 @@ -name: build-kernel-deb - -on: - workflow_dispatch: - inputs: - kernel-variant: - description: 'Kernel variant identifier' - required: true - default: 'qcom-next' - type: string - suite: - description: 'Target suite' - required: true - default: 'trixie' - type: string - ref-strategy: - description: 'Kernel ref strategy' - required: true - default: 'latest_tag' - type: choice - options: - - latest_tag - - branch_tip - - pinned_ref - kernel-branch: - description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' - required: false - default: 'qcom-next' - tag-pattern: - description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' - required: false - default: 'qcom-next-*' - kernel-url: - description: 'Advanced: custom kernel repository URL (empty uses qualcomm-linux/kernel)' - required: false - default: '' - srcpkg: - description: 'Advanced: Debian source package name' - required: false - default: 'linux-qcom-next' - binpkg: - description: 'Advanced: kernel image metapackage name' - required: false - default: 'linux-image-qcom-next' - kernel-config: - description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - required: false - default: '' - dkms: - description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - required: false - default: '' - debian-version-stub: - description: 'Advanced: Debian version stub; the selected suite''s mapped suffix and a Daily-style trailing ~ are applied automatically' - required: false - default: '0qli' - localversion: - description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' - required: false - default: '' - kver-extra: - description: 'Advanced: extra package-version suffix (e.g. -ci42)' - required: false - default: '' - debug-build: - description: 'Advanced: enable the debug build configuration' - type: boolean - required: false - default: false - qcom-next-pr: - description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' - type: string - required: false - default: '' - kernel-topics-pr: - description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' - type: string - required: false - default: '' - - # Called by daily.yml and release.yml as a reusable workflow. - workflow_call: - inputs: - kernel-variant: - description: 'Kernel variant identifier' - type: string - default: 'qcom-next' - suite: - description: 'Target suite (trixie, forky, noble, questing, resolute, sid)' - type: string - default: 'trixie' - kernel-branch: - description: 'Source branch or pinned tag used by branch_tip or pinned_ref' - type: string - default: 'qcom-next' - git-clone: - description: 'Kernel repository URL' - type: string - default: 'https://github.com/qualcomm-linux/kernel' - ref-strategy: - description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' - type: string - default: 'latest_tag' - tag-pattern: - description: 'Tag glob used when ref-strategy=latest_tag' - type: string - default: 'qcom-next-*' - srcpkg: - description: 'Source package name' - type: string - default: 'linux-qcom-next' - binpkg: - description: 'Binary metapackage name' - type: string - default: 'linux-image-qcom-next' - kernel-config: - description: 'Extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - type: string - default: '' - dkms: - description: 'Out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - type: string - default: '' - debian-revision: - description: 'Debian revision component of the package version' - type: string - default: '0qli~' - localversion: - description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' - type: string - default: '' - kver-extra: - description: 'Extra suffix appended to the package version' - type: string - default: '' - debusine-parent-workspace: - description: 'Parent Debusine workspace for CI child workspace creation' - type: string - default: '' - target-workspace: - description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' - type: string - default: '' - secrets: - DEBUSINE_USER: - description: 'Debusine account used to submit CI and release builds' - required: false - DEBUSINE_TOKEN: - description: 'Debusine token for the CI child workspace' - required: false - DEBUSINE_RELEASE_TOKEN: - description: 'Debusine token for package-publish promotion to the release target workspace' - required: false - -permissions: - contents: read - packages: read - -jobs: - # --------------------------------------------------------------------------- - # resolve: classify the target suite as 'debian' or 'ubuntu'. - # --------------------------------------------------------------------------- - resolve: - name: Resolve suite family - runs-on: ubuntu-latest - outputs: - family: ${{ steps.classify.outputs.family }} - steps: - - name: Classify suite - id: classify - shell: bash - env: - SUITE_INPUT: ${{ inputs.suite || 'trixie' }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} - run: | - suite="$SUITE_INPUT" - kernel_variant="$KERNEL_VARIANT_INPUT" - [[ "$kernel_variant" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "ERROR: kernel variant must use lowercase letters, digits, and internal hyphens" >&2 - exit 1 - } - case "$suite" in - trixie|forky|sid|unstable|bookworm) - family=debian ;; - *) - family=ubuntu ;; - esac - echo "family=$family" >> "$GITHUB_OUTPUT" - echo "Suite '$suite' classified as family=$family" - - # --------------------------------------------------------------------------- - # prepare: clone kernel source, inject packaging, run prepare-source.sh. - # --------------------------------------------------------------------------- - prepare: - name: Prepare kernel source - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - # TEMPORARY, with the DKMS override in "Prepare source" below. - needs: resolve - - env: - KERNEL_URL: ${{ inputs.git-clone || inputs.kernel-url || 'https://github.com/qualcomm-linux/kernel' }} - DISTRO: ${{ inputs.suite || 'trixie' }} - # For Debian-family suites (trixie, forky, sid) use trixie as the docker - # image distro: docker-pkg-build and pkg-builder only support trixie for - # Debian. Debusine handles the actual suite-specific build internally. - # For Ubuntu-family suites (noble, questing, resolute) use the real suite. - DOCKER_DISTRO: ${{ (inputs.suite == 'forky' || inputs.suite == 'sid' || inputs.suite == 'unstable') && 'trixie' || inputs.suite || 'trixie' }} - - defaults: - run: - shell: bash - - steps: - - name: Checkout pkg-linux-qcom - # debian/ and ci/ live on the same branch, so one checkout supplies - # both the packaging and resolve-kernel-ref.sh, derive-localversion.sh, - # derive-debian-revision.sh and the suite_suffix_mapping. - uses: actions/checkout@v4 - - - name: Checkout docker-pkg-build - uses: actions/checkout@v4 - with: - repository: qualcomm-linux/docker-pkg-build - ref: main - path: docker-pkg-build - - - name: Build docker image - run: | - ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" - docker image ls - - - name: Resolve kernel ref - id: kernel-ref - env: - BRANCH_INPUT: ${{ inputs.kernel-branch || 'qcom-next' }} - REF_STRATEGY_INPUT: ${{ inputs.ref-strategy || 'latest_tag' }} - TAG_PATTERN_INPUT: ${{ inputs.tag-pattern || 'qcom-next-*' }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} - run: | - set -euo pipefail - BRANCH="$BRANCH_INPUT" - REF_STRATEGY="$REF_STRATEGY_INPUT" - TAG_PATTERN="$TAG_PATTERN_INPUT" - - case "$REF_STRATEGY" in - latest_tag) - KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ - --url "$KERNEL_URL" \ - --latest-tag "$TAG_PATTERN") - ;; - branch_tip|pinned_ref) - KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh --url "$KERNEL_URL" --ref "$BRANCH") - ;; - *) - echo "ERROR: unsupported ref strategy: $REF_STRATEGY" >&2 - exit 1 - ;; - esac - - echo "KERNEL_REF=$KERNEL_REF" >> "$GITHUB_ENV" - echo "Kernel variant: $KERNEL_VARIANT_INPUT" - echo "Ref strategy: $REF_STRATEGY" - echo "Resolved kernel ref: $KERNEL_REF" - - - name: Clone kernel source - run: | - KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" - echo "KERNEL_DIR=$KERNEL_DIR" >> "$GITHUB_ENV" - echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." - git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ - "$KERNEL_URL" "$KERNEL_DIR" - KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) - echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" - echo "Kernel HEAD SHA: $KERNEL_SHA" - - - name: Derive LOCALVERSION - env: - LOCALVERSION_INPUT: ${{ inputs.localversion }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} - run: | - set -euo pipefail - OVERRIDE_LV="$LOCALVERSION_INPUT" - if [[ -n "$OVERRIDE_LV" ]]; then - echo "Using explicit LOCALVERSION override: $OVERRIDE_LV" - echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" - else - LOCALVERSION=$(ci/scripts/derive-localversion.sh \ - --variant "$KERNEL_VARIANT_INPUT" \ - --ref "$KERNEL_REF" \ - --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") - echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" - echo "Derived LOCALVERSION: $LOCALVERSION" - fi - - - name: Resolve Debian revision - # workflow_call callers (daily.yml, release.yml) pass an already - # fully-derived debian-revision from resolve-matrix.sh. Direct - # workflow_dispatch runs pass only debian-version-stub and get the - # selected suite's mapped suffix applied automatically, always using - # Daily semantics since direct dispatch is build-only and - # non-promoting. - env: - DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} - DEBIAN_VERSION_STUB_INPUT: ${{ inputs.debian-version-stub || '0qli' }} - run: | - set -euo pipefail - INPUT_REVISION="$DEBIAN_REVISION_INPUT" - if [[ -n "$INPUT_REVISION" ]]; then - DEBIAN_REVISION="$INPUT_REVISION" - echo "Using matrix-derived Debian revision: $DEBIAN_REVISION" - else - DEBIAN_REVISION=$(ci/scripts/derive-debian-revision.sh \ - --stub "$DEBIAN_VERSION_STUB_INPUT" \ - --suite "$DISTRO" \ - --delivery-type Daily \ - --matrix-file ci/build-matrix.json) - echo "Derived Debian revision for direct dispatch: $DEBIAN_REVISION" - fi - echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" - - - name: Merge qcom-next PR patches - if: ${{ inputs.qcom-next-pr != '' }} - env: - QCOM_NEXT_PR: ${{ inputs.qcom-next-pr }} - run: | - cd "$KERNEL_DIR" - echo "Merging qcom-next PR(s): $QCOM_NEXT_PR" - for pr in $QCOM_NEXT_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid qcom-next PR number: $pr"; exit 1; } - echo "::group::Merging qcom-next PR #$pr" - git fetch --no-tags origin "pull/$pr/head:pr-$pr" - if ! git merge --no-ff --no-commit "pr-$pr"; then - echo "ERROR: Merge conflict while merging PR #$pr. Aborting." - git merge --abort || true - exit 1 - fi - if ! git diff --cached --quiet; then - git commit -m "Merged qcom-next PR #$pr" - echo "PR #$pr merged successfully." - else - echo "PR #$pr already present; nothing to merge." - fi - echo "::endgroup::" - done - - - name: Apply kernel-topics PR patches - if: ${{ inputs.kernel-topics-pr != '' }} - env: - KERNEL_TOPICS_PR: ${{ inputs.kernel-topics-pr }} - run: | - cd "$KERNEL_DIR" - echo "Applying kernel-topics PR patch(es): $KERNEL_TOPICS_PR" - for pr in $KERNEL_TOPICS_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid kernel-topics PR number: $pr"; exit 1; } - echo "::group::Applying kernel-topics PR #$pr" - wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - if ! git am "$pr.patch"; then - echo "ERROR: Patch application failed for PR #$pr. Aborting." - git am --abort || true - exit 1 - fi - echo "PR #$pr applied successfully." - echo "::endgroup::" - done - - - name: Prepare source - env: - SRCPKG_INPUT: ${{ inputs.srcpkg || 'linux-qcom-next' }} - BINPKG_INPUT: ${{ inputs.binpkg || 'linux-image-qcom-next' }} - KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} - KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} - DKMS_INPUT: ${{ inputs.dkms }} - # TEMPORARY, consumed by the DKMS override below. - SUITE_FAMILY: ${{ needs.resolve.outputs.family }} - DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} - run: | - # No -x here: keep xtrace off for this step as it assembles the - # prepare-source.sh argument list which may include sensitive paths. - set -euo pipefail - - DKMS_LIST="$DKMS_INPUT" - - # TEMPORARY: camx and iris-vpu are bundled on non-Ubuntu builds only. - # A listed module is a presence contract, so an Ubuntu leg that asked - # for them would hard-fail rather than quietly ship without them. - # - # This is delivery policy and belongs in ci/build-matrix.json, which - # cannot express a per-suite dkms list yet. Revert this override — - # and the "needs: resolve"/SUITE_FAMILY plumbing that feeds it — once - # the matrix can. - if [[ "$SUITE_FAMILY" == "ubuntu" ]]; then - DKMS_LIST="kgsl" - echo "Ubuntu build: DKMS list overridden to '$DKMS_LIST' (was '$DKMS_INPUT')" - fi - - ARGS=( - --source-dir "$KERNEL_DIR" - --distro "$DISTRO" - --srcpkg "$SRCPKG_INPUT" - --binpkg "$BINPKG_INPUT" - --debian-revision "$DEBIAN_REVISION" - ) - - [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") - [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") - [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") - # Always passed, unlike the optional inputs above: an empty list is a - # genuine "bundle nothing" rather than a request for some default. - ARGS+=(--dkms "$DKMS_LIST") - [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) - - docker run -i --rm \ - -v "$PWD:$PWD" --workdir="$PWD" \ - "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ - ./prepare-source.sh "${ARGS[@]}" - - - name: Upload prepared source tree - run: | - tar czf /tmp/kernel-srcpkg.tar.gz \ - -C "$(dirname "$KERNEL_DIR")" \ - "$(basename "$KERNEL_DIR")" - - - name: Upload prepared source tree artifact - uses: actions/upload-artifact@v4 - with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} - path: /tmp/kernel-srcpkg.tar.gz - if-no-files-found: error - retention-days: 7 - - # --------------------------------------------------------------------------- - # debusine-build: Debian suites only (trixie, forky, sid). - # --------------------------------------------------------------------------- - debusine-build: - name: Build in Debusine - needs: [resolve, prepare] - if: ${{ needs.resolve.outputs.family == 'debian' }} - uses: ./.github/workflows/build-kernel-debusine.yml - with: - suite: ${{ inputs.suite || 'trixie' }} - kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} - srcpkg: ${{ inputs.srcpkg || 'linux-qcom-next' }} - debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} - target-workspace: ${{ inputs.target-workspace || '' }} - secrets: - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - - # --------------------------------------------------------------------------- - # ubuntu-build: Ubuntu suites only (noble, questing, resolute). - # --------------------------------------------------------------------------- - ubuntu-build: - name: Build (Ubuntu, docker) - needs: [resolve, prepare] - if: ${{ needs.resolve.outputs.family == 'ubuntu' }} - uses: ./.github/workflows/build-kernel-ubuntu.yml - with: - distro: ${{ inputs.suite || 'resolute' }} - kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml new file mode 100644 index 00000000..6049370c --- /dev/null +++ b/.github/workflows/build-kernel-debian.yml @@ -0,0 +1,491 @@ +name: build-kernel-debian + +# Build a kernel package for a Debian-family suite, publish it to S3, and +# promote it into a Debusine workspace when the caller names one. +# +# Called per matrix entry by daily.yml and pr-build.yml, which select the +# Debian entries with resolve-matrix.py --family debian and call only this +# workflow for them. Nothing here is conditional on the suite: a caller that +# reaches this file has already decided the build belongs on the Debusine +# path. +# +# workflow_call only. One run of this workflow is one matrix entry, and a +# reusable workflow cannot fan itself out over several, so manual builds are +# dispatched from daily.yml -- which resolves the entries and calls this once +# per Debian entry it selected. That also keeps every value describing a build +# in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. +# +# Publishing happens here and nowhere else. A caller that names a +# target-workspace has this run's own build promoted into it, out of the +# ephemeral CI workspace it was built in, so the archive carries the artifact +# this run produced and tested. There is no later step that could publish +# something else, and nothing rebuilds a ref that has already been built. +# +# Jobs: +# prepare (self-hosted): clone, inject debian/, build the source package, +# and decide whether the version it produces is already published. +# build (debusine-pkg-builder container): submit to Debusine. +# publish (self-hosted): download the .deb files and upload them to S3. +# promote (debusine-pkg-builder container): copy the result into +# target-workspace. Skipped entirely when no workspace was named. + +on: + workflow_call: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + required: true + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + required: true + suite: + description: 'Target Debian suite (trixie, forky, sid)' + type: string + required: true + kernel-branch: + description: 'Source branch or pinned tag used by branch_tip or pinned_ref' + type: string + required: true + git-clone: + description: 'Kernel repository URL' + type: string + required: true + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + type: string + required: true + tag-pattern: + description: 'Tag glob used when ref-strategy=latest_tag' + type: string + default: 'qcom-next-*' + srcpkg: + description: 'Source package name' + type: string + required: true + binpkg: + description: 'Binary metapackage name' + type: string + required: true + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + required: true + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + type: string + default: '' + extra-build-dep-workspaces: + description: > + Workspaces the build resolves its Build-Depends from, space + separated. The caller decides, because it follows from what becomes + of the result: the nightly is what a release promotes, so it reads + released archives only, and a PR build may legitimately need a + package that has reached staging and no further. + type: string + default: 'qli' + target-workspace: + description: > + Debusine workspace to promote the built package into, empty to + promote nowhere. The caller decides: the nightly build names the + staging workspace, and pr-build.yml leaves it empty because a pull + request's kernel must not reach an archive. + type: string + default: '' + promote-environment: + description: > + GitHub environment the promote job runs in, and so which approvals + and which credentials stand between a build and the archive it + publishes into. Defaults to Staging, where the nightly promotes + unattended. release.yml passes Production, because a release reaches + an archive people install from and an unreviewed one must not. + type: string + default: 'Staging' + secrets: + DEBUSINE_USER: + description: 'Debusine account used to submit CI builds' + required: false + DEBUSINE_TOKEN: + description: 'Debusine token for the CI child workspace' + required: false + +permissions: + contents: read + packages: read + +jobs: + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh, and + # report the version that produces. + # + # The version is decided here, by the ref this run resolved and the revision + # it was given, so this is the first point at which the run can tell whether + # there is anything new to build. When the caller named a target workspace + # and that version is already in it, every later job is skipped: the nightly + # build of a tag that has not moved would otherwise rebuild the same version + # and then fail promoting it as a duplicate. + # --------------------------------------------------------------------------- + prepare: + name: Generate source package + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder + outputs: + srcpkg_version: ${{ steps.prepare.outputs.srcpkg_version }} + already_published: ${{ steps.published.outputs.already_published }} + steps: + # debian/ and ci/ live on the same branch, so one checkout supplies both + # the packaging and the scripts, the delivery matrix they read, and the + # prepare-kernel-source action below. + # + # An empty ref leaves actions/checkout on github.sha, the commit this run + # was dispatched from or the one the calling workflow runs at. + - name: Checkout pkg-linux-qcom + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Prepare kernel source + id: prepare + uses: ./.github/actions/prepare-kernel-source + with: + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: debian + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} + + # Only needed to ask the target workspace what it already has, so it is + # checked out only when there is a target workspace to ask. + - name: Checkout debusine-action helpers + if: ${{ inputs.target-workspace != '' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + # The archive is the authority on whether there is anything to do. A + # rerun, a retry, and a night on which the tracked tag did not move all + # generate the version that is already published, because the version is + # a function of the resolved ref and the revision -- so asking here is + # what turns "nothing new upstream" into a skipped run rather than a + # rebuild that fails promoting a duplicate. + # + # Two ways of not getting an answer, deliberately treated differently. + # A workspace whose signing key cannot be fetched is a configuration + # problem -- it does not exist, or this token cannot see it -- and fails + # the run, because every later night would fail the same way and silence + # would hide it. An index that cannot be read once the workspace is + # reachable is reported as not having the version, which builds and lets + # the promotion fail loudly. Neither is allowed to look like a + # successful night on which nothing needed doing. + - name: Check the target workspace for this version + id: published + if: ${{ inputs.target-workspace != '' }} + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_WORKSPACE: ${{ inputs.target-workspace }} + SUITE: ${{ inputs.suite }} + SRCPKG: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ steps.prepare.outputs.srcpkg_version }} + ORIG_FILE: ${{ steps.prepare.outputs.orig_file }} + ORIG_SHA256: ${{ steps.prepare.outputs.orig_sha256 }} + run: | + # No -x: DEBUSINE_TOKEN is in env, keep xtrace off. + set -euo pipefail + debusine-action/lib/generate-apt-config || { + echo "::error::Cannot read $DEBUSINE_WORKSPACE. Check that the workspace exists and that DEBUSINE_TOKEN may read it." + exit 1 + } + sudo apt-get install -y --no-install-recommends devscripts + chdist create target-workspace + rm -f ~/.chdist/target-workspace/etc/apt/sources.list + install -d ~/.chdist/target-workspace/etc/apt/sources.list.d + install -d ~/.chdist/target-workspace/etc/apt/auth.conf.d + install -m 0644 debusine-ci.sources \ + ~/.chdist/target-workspace/etc/apt/sources.list.d/ + install -m 0600 debusine-ci-auth.conf \ + ~/.chdist/target-workspace/etc/apt/auth.conf.d/ + + if ! chdist apt-get target-workspace update; then + echo "::warning::Could not read $DEBUSINE_WORKSPACE; treating $SRCPKG $SRCPKG_VERSION as unpublished." + echo "already_published=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + SOURCES=$(chdist apt-cache target-workspace showsrc "$SRCPKG" 2>/dev/null || true) + + # Whether there is anything to do at all, before anything is + # inspected about what would be built: a run that publishes nothing + # uploads no files, so nothing it holds can conflict with the + # archive. + if grep -qxF "Version: $SRCPKG_VERSION" <<<"$SOURCES"; then + echo "$SRCPKG $SRCPKG_VERSION is already in $DEBUSINE_WORKSPACE." + echo "Nothing new to build; the rest of this run is skipped." \ + >> "$GITHUB_STEP_SUMMARY" + echo "already_published=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "$SRCPKG $SRCPKG_VERSION is not in $DEBUSINE_WORKSPACE yet." + echo "already_published=false" >> "$GITHUB_OUTPUT" + + # This version is going to be built and promoted, so the orig it + # carries has to agree with any the archive already holds under that + # name. It can hold one: a packaging rebuild publishes a second + # Debian revision of one upstream version, and the archive requires + # the two revisions to share the file. The orig is a function of the + # kernel commit, so a stanza naming it with another checksum means + # two builds of one commit disagreed. That is rejected at promotion + # as a file conflict, or worse accepted somewhere that does not + # check, so it fails here with both checksums instead. + # + # This suite only: apt indexes are per suite, so an orig published + # under another suite of the workspace is not seen here. The + # reproducibility of the tarball is what covers that case. + ARCHIVE_SHA256=$(awk -v orig="$ORIG_FILE" \ + '$1 ~ /^[0-9a-f]{64}$/ && $3 == orig { print $1 }' <<<"$SOURCES" | sort -u) + if [[ -n "$ARCHIVE_SHA256" && "$ARCHIVE_SHA256" != "$ORIG_SHA256" ]]; then + echo "::error::$DEBUSINE_WORKSPACE $SUITE already holds $ORIG_FILE with sha256 $ARCHIVE_SHA256, but this run built it as $ORIG_SHA256." + echo "::error::An orig tarball is a function of the commit it names; find what differs between the two builds before publishing either." + exit 1 + fi + [[ -z "$ARCHIVE_SHA256" ]] \ + || echo "$ORIG_FILE in $DEBUSINE_WORKSPACE $SUITE matches this build ($ORIG_SHA256)." + + # --------------------------------------------------------------------------- + # build: generate the Debian source package and submit it to Debusine. + # + # Skipped, with everything after it, when prepare found this version already + # published in the target workspace. + # --------------------------------------------------------------------------- + build: + name: Build package (Debusine) + needs: prepare + if: ${{ needs.prepare.outputs.already_published != 'true' }} + runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder + environment: Staging + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + outputs: + workspace: ${{ steps.debusine.outputs.workspace }} + srcpkg_version: ${{ steps.debusine.outputs.srcpkg_version }} + steps: + # Only for the debusine-build action itself: the package being built + # comes from the source-package artifact, not from this checkout. + - name: Checkout pkg-linux-qcom + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build in Debusine + id: debusine + uses: ./.github/actions/debusine-build + with: + suite: ${{ inputs.suite }} + flavour: ${{ inputs.flavour }} + debusine-host: ${{ vars.DEBUSINE_HOST }} + debusine-scope: ${{ vars.DEBUSINE_SCOPE }} + debusine-user: ${{ secrets.DEBUSINE_USER }} + debusine-token: ${{ secrets.DEBUSINE_TOKEN }} + debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} + extra-build-dep-workspaces: ${{ inputs.extra-build-dep-workspaces }} + + # --------------------------------------------------------------------------- + # publish: download the .deb files from Debusine and upload them to S3. + # Runs on the self-hosted runner, which has direct IAM access. + # --------------------------------------------------------------------------- + publish: + name: Publish to S3 + needs: build + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Generate Debusine apt configuration + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_WORKSPACE: ${{ needs.build.outputs.workspace }} + SUITE: ${{ inputs.suite }} + run: | + debusine-action/lib/generate-apt-config + + - name: Download .deb packages from Debusine workspace + run: | + set -euxo pipefail + sudo apt-get install -y --no-install-recommends devscripts + mkdir -p "$GITHUB_WORKSPACE/deb-artifacts" + chdist create debusine-workspace + rm -f ~/.chdist/debusine-workspace/etc/apt/sources.list + install -d ~/.chdist/debusine-workspace/etc/apt/sources.list.d + install -d ~/.chdist/debusine-workspace/etc/apt/auth.conf.d + install -m 0644 debusine-ci.sources \ + ~/.chdist/debusine-workspace/etc/apt/sources.list.d/ + install -m 0600 debusine-ci-auth.conf \ + ~/.chdist/debusine-workspace/etc/apt/auth.conf.d/ + chdist apt-get debusine-workspace update + packages=$(chdist apt-cache debusine-workspace search . | awk '{print $1}' | tr '\n' ' ') + [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace"; exit 1; } + echo "Packages to download: $packages" + cd "$GITHUB_WORKSPACE/deb-artifacts" + # shellcheck disable=SC2086 + chdist apt-get debusine-workspace download $packages + + - name: Upload .deb packages to S3 + # Keep flavour and suite in the destination so concurrent matrix legs + # cannot overwrite or mix package outputs. Consumers must select the + # flavour and suite they intend to install. + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 + with: + s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} + path: deb-artifacts + destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + + # The source package the binaries were built from, kept beside them: + # the .changes records every checksum, and the .dsc is enough to + # rebuild the packages anywhere. The Actions artifact of the same + # files expires; this does not. + - name: Download source package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} + path: source-package + + - name: Upload source package to S3 + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 + with: + s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} + path: source-package + destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/source/ + + # --------------------------------------------------------------------------- + # promote: copy the built package from the ephemeral CI workspace into the + # workspace the caller named, making it installable from that archive. + # + # Promotion is in-run by necessity, not by preference: lib/build names the + # CI workspace after the run that created it + # (-gh----) and it does not outlive it, so + # the only moment its contents can be published is while the run still holds + # it. That is why release.yml builds the ref it releases rather than + # promoting a nightly afterwards -- by then there would be nothing left to + # promote from. + # + # The if: is the whole of the decision: a caller that named no workspace + # promotes nowhere. pr-build.yml is that caller, so a pull request's kernel + # never reaches an archive. Which archive, and what stands in front of it, + # is the caller's too: promote-environment carries the approval gate, so a + # release into qli can require one where the nightly into staging does not. + # --------------------------------------------------------------------------- + promote: + name: Promote to ${{ inputs.target-workspace }} + # build alone: it already needs prepare, so a run prepare skipped as + # already published skips the build and this job with it. + needs: build + if: ${{ inputs.target-workspace != '' }} + runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder + environment: ${{ inputs.promote-environment }} + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Promote packages to the target workspace + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} + DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} + SRCPKG_NAME: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} + SUITE: ${{ inputs.suite }} + run: | + set -eu + debusine-action/lib/release + + - name: Note the promoted version + env: + SRCPKG_NAME: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} + TARGET_WORKSPACE: ${{ inputs.target-workspace }} + run: | + echo "Promoted $SRCPKG_NAME $SRCPKG_VERSION to $TARGET_WORKSPACE" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/build-kernel-debusine.yml b/.github/workflows/build-kernel-debusine.yml deleted file mode 100644 index 7b8cc17a..00000000 --- a/.github/workflows/build-kernel-debusine.yml +++ /dev/null @@ -1,242 +0,0 @@ -name: build-debusine - -# Reusable workflow: Debian kernel build via Debusine, then publish. -# -# Called by build-kernel-deb.yml for Debian-family suites (trixie, forky, sid). -# The kernel source is already prepared by the caller's prepare job and shared -# as the kernel-srcpkg artifact. Artifact and workspace identity include both -# kernel_variant and suite so parallel variants cannot share inputs or outputs. -# -# Publish path is determined by target-workspace: -# target-workspace == '' -> daily path: download .deb files, upload to S3. -# target-workspace != '' -> release path: promote from CI workspace to the -# target Debusine workspace via debusine-action lib/release. -# -# Jobs: -# build (debusine-pkg-builder container): generates .dsc, submits to Debusine. -# publish (self-hosted runner): S3 upload (daily path only). -# release (debusine-pkg-builder container): Debusine promotion (release path only). - -on: - workflow_call: - inputs: - suite: - description: 'Target Debian suite (trixie, forky, sid)' - type: string - required: true - kernel-variant: - description: 'Kernel variant identifier used to isolate artifacts and Debusine workspaces' - type: string - required: true - srcpkg: - description: 'Source package name (e.g. linux-qcom-next)' - type: string - default: 'linux-qcom-next' - debusine-parent-workspace: - description: 'Parent Debusine workspace for CI child workspace creation' - type: string - default: 'qli-ci' - target-workspace: - description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' - type: string - default: '' - secrets: - DEBUSINE_USER: - required: true - DEBUSINE_TOKEN: - required: true - DEBUSINE_RELEASE_TOKEN: - required: false - -permissions: - contents: read - packages: read - -env: - DEBUSINE_ACTION_REF: main - -jobs: - # --------------------------------------------------------------------------- - # build: generate Debian source package and submit to Debusine. - # --------------------------------------------------------------------------- - build: - name: Build (Debusine) - runs-on: ubuntu-latest - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - outputs: - workspace: ${{ steps.build-debusine.outputs.workspace }} - workspace_url: ${{ steps.build-debusine.outputs.workspace_url }} - srcpkg_version: ${{ steps.generate-srcpkg.outputs.srcpkg_version }} - defaults: - run: - shell: bash - environment: ${{ inputs.target-workspace != '' && 'Production' || 'Staging' }} - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@v5 - with: - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Download prepared source tree - uses: actions/download-artifact@v4 - with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} - path: /tmp/srcpkg-artifact - - - name: Extract prepared source tree - run: | - mkdir srcpkg - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C srcpkg --strip-components=1 - - - name: Generate source package - id: generate-srcpkg - env: - SUITE: ${{ inputs.suite }} - DEBUSINE_ASSEMBLE_ORIG: "true" - run: | - SUITE="$SUITE" debusine-action/lib/generate-source-package - - - name: Build in Debusine - id: build-debusine - env: - GITHUB_REPOSITORY_ID: ${{ github.repository_id }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_INDEX: ${{ inputs.kernel-variant }}-${{ inputs.suite }} - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} - SUITE: ${{ inputs.suite }} - EXTRA_BUILD_DEP_WORKSPACES: qli - run: | - # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. - set -euo pipefail - debusine-action/lib/build - workspace=$(sed -n 's/^workspace=//p' "$GITHUB_OUTPUT") - echo "workspace_url=https://${DEBUSINE_HOST}/${DEBUSINE_SCOPE}/${workspace}/" >> "$GITHUB_OUTPUT" - - - name: Note Debusine workspace URL - env: - WORKSPACE_URL: ${{ steps.build-debusine.outputs.workspace_url }} - run: | - echo "Debusine Workspace URL: $WORKSPACE_URL" >> "$GITHUB_STEP_SUMMARY" - - # --------------------------------------------------------------------------- - # publish: download .deb files from Debusine and upload to S3. - # Daily path only (target-workspace is empty). - # --------------------------------------------------------------------------- - publish: - name: Publish to S3 - needs: build - if: ${{ needs.build.result == 'success' && inputs.target-workspace == '' }} - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@v5 - with: - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Generate Debusine apt configuration - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_WORKSPACE: ${{ needs.build.outputs.workspace }} - SUITE: ${{ inputs.suite }} - run: | - debusine-action/lib/generate-apt-config - - - name: Download .deb packages from Debusine workspace - run: | - set -euxo pipefail - sudo apt-get install -y --no-install-recommends devscripts - mkdir -p "$GITHUB_WORKSPACE/deb-artifacts" - chdist create debusine-workspace - rm -f ~/.chdist/debusine-workspace/etc/apt/sources.list - install -d ~/.chdist/debusine-workspace/etc/apt/sources.list.d - install -d ~/.chdist/debusine-workspace/etc/apt/auth.conf.d - install -m 0644 debusine-ci.sources \ - ~/.chdist/debusine-workspace/etc/apt/sources.list.d/ - install -m 0600 debusine-ci-auth.conf \ - ~/.chdist/debusine-workspace/etc/apt/auth.conf.d/ - chdist apt-get debusine-workspace update - packages=$(chdist apt-cache debusine-workspace search . | awk '{print $1}' | tr '\n' ' ') - [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace"; exit 1; } - echo "Packages to download: $packages" - cd "$GITHUB_WORKSPACE/deb-artifacts" - # shellcheck disable=SC2086 - chdist apt-get debusine-workspace download $packages - - - name: Upload .deb packages to S3 - uses: qualcomm-linux/upload-private-artifact-action@aws-v4 - with: - s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} - path: deb-artifacts - destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ - - # --------------------------------------------------------------------------- - # release: promote packages from CI workspace to target Debusine workspace. - # Release path only (target-workspace is non-empty). - # --------------------------------------------------------------------------- - release: - name: Release to Debusine - needs: build - if: ${{ needs.build.result == 'success' && inputs.target-workspace != '' }} - runs-on: ubuntu-latest - environment: Production - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - with: - persist-credentials: false - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Promote packages to target workspace - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} - DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} - SRCPKG_NAME: ${{ inputs.srcpkg }} - SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} - SUITE: ${{ inputs.suite }} - run: | - set -ex - debusine-action/lib/release diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index f62317ec..20b2acf9 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -1,99 +1,179 @@ name: build-kernel-ubuntu -# Reusable workflow: Ubuntu kernel build via build-kernel.sh. +# Build a kernel package for an Ubuntu-family suite and publish it to S3. # -# Called by build-kernel-deb.yml when the target suite is Ubuntu-family -# (noble, questing, resolute). The kernel source has already been cloned, -# patched, and prepared (debian/ injected, debian/control + debian/changelog -# generated) by the prepare job in the caller workflow. This workflow -# downloads the kernel-srcpkg artifact and runs build-kernel.sh --skip-prepare -# to go directly to the build step. +# Called per matrix entry by daily.yml and pr-build.yml, which select the +# Ubuntu entries with resolve-matrix.py --family ubuntu and call only this +# workflow for them. The package is built in a suite-matched pkg-builder +# container on the self-hosted runner rather than in Debusine, which builds +# only the Debian family. # -# The artifact is shared from the caller via github.run_id. Build and S3 upload -# run on the self-hosted runner, which has direct IAM access. +# Ubuntu has no release path: promotion runs through Debusine, so a Release +# entry for an Ubuntu suite is rejected by resolve-matrix.py rather than built +# here and quietly not promoted. +# +# workflow_call only. One run of this workflow is one matrix entry, and a +# reusable workflow cannot fan itself out over several, so manual builds are +# dispatched from daily.yml -- which resolves the entries and calls this once +# per Ubuntu entry it selected. That also keeps every value describing a build +# in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. +# +# One job, on the self-hosted runner: clone, inject debian/, build the source +# package, build the .deb files from it, and upload them to S3. on: workflow_call: inputs: - distro: - description: Target Ubuntu suite (noble, questing, resolute) + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + required: true + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + required: true + suite: + description: 'Target Ubuntu suite (noble, resolute)' + type: string + required: true + kernel-branch: + description: 'Source branch or pinned tag used by branch_tip or pinned_ref' + type: string + required: true + git-clone: + description: 'Kernel repository URL' + type: string + required: true + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + type: string + required: true + tag-pattern: + description: 'Tag glob used when ref-strategy=latest_tag' + type: string + default: 'qcom-next-*' + srcpkg: + description: 'Source package name' type: string required: true - kernel-variant: - description: 'Kernel variant identifier used to isolate prepared-source artifacts and published packages' + binpkg: + description: 'Binary metapackage name' type: string required: true + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + required: true + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' permissions: contents: read packages: read jobs: + # --------------------------------------------------------------------------- + # build: clone, prepare, build the source package, build binaries from it, + # and publish them. + # + # One job rather than a prepare job and a build job, because every step + # runs on the same runner label: the source package the prepare action + # writes is consumed here from the workspace it was written into, with no + # artifact uploaded only to be downloaded again by the next job. The action + # still uploads it, as the record of what was built. + # + # Named for where the build happens, as the Debian workflow's Build package + # (Debusine) is: this one runs in a suite-matched pkg-builder container on + # the runner rather than being submitted to a build service. + # --------------------------------------------------------------------------- build: - name: Build kernel package + name: Build package (Docker) runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder env: - DISTRO: ${{ inputs.distro }} + DISTRO: ${{ inputs.suite }} defaults: run: shell: bash - steps: - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 - - # Pinned to @main while native kernel-build support is still landing - # upstream in qualcomm-linux/docker-pkg-build. - - name: Checkout docker-pkg-build - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: qualcomm-linux/docker-pkg-build - ref: main - path: docker-pkg-build + persist-credentials: false - - name: Build docker image - run: | - # --rebuild forces a fresh suite-matched pkg-builder image. - ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DISTRO" - docker image ls - - - name: Download prepared source tree - uses: actions/download-artifact@v4 + # Also pulls, or builds from a docker-pkg-build checkout, the suite's + # pkg-builder image -- the image the binary build below runs sbuild in. + - name: Prepare kernel source + uses: ./.github/actions/prepare-kernel-source with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.distro }} - path: /tmp/srcpkg-artifact - - - name: Extract prepared source tree - run: | - # tar.gz preserves the execute bits that upload-artifact's zip would - # strip (see README); --strip-components=1 drops the archive's top dir. - mkdir -p kernel-source - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C kernel-source --strip-components=1 - echo "KERNEL_DIR=$GITHUB_WORKSPACE/kernel-source" >> "$GITHUB_ENV" + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: ubuntu + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} - name: Build kernel package run: | - # Kernel source was cloned, patched, and prepared by the caller's - # prepare job. Pass --local-source and --skip-prepare so - # build-kernel.sh goes directly to the build step. - # debian/control, debian/changelog, and all config fragments are - # already baked into the extracted source tree; no re-derivation - # of LOCALVERSION, kver-extra, or debug config is needed here. + # The source package sits where the action wrote it. --dsc hands + # it to sbuild in the suite's pkg-builder image, so what is built + # here is the .dsc the run reported, exactly as Debusine builds it + # for the Debian family. Nothing is cloned or prepared again, and + # build-kernel.sh finds the docker-pkg-build checkout the action + # leaves behind when it had to build the image itself. + set -euo pipefail + DSC=$(ls source-package/*.dsc) + [[ $(wc -l <<<"$DSC") -eq 1 ]] || { + echo "::error::Expected exactly one .dsc in source-package/, found: $DSC" + exit 1 + } ./build-kernel.sh \ --build-mode docker \ --distro "$DISTRO" \ - --local-source "$KERNEL_DIR" \ - --skip-prepare \ - --docker-build "$GITHUB_WORKSPACE/docker-pkg-build/docker_deb_build.py" + --dsc "$DSC" - name: Upload .deb packages to S3 # build-kernel.sh outputs to kernel-build// by default. - # Keep kernel variant and suite in the destination so concurrent matrix - # legs cannot overwrite or mix package outputs. Consumers must select - # the variant and suite they intend to install. - uses: qualcomm-linux/upload-private-artifact-action@aws-v4 + # Keep flavour and suite in the destination so concurrent matrix legs + # cannot overwrite or mix package outputs. Consumers must select the + # flavour and suite they intend to install. + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 + with: + s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} + path: kernel-build/${{ inputs.suite }} + destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + + # The source package the binaries were built from, kept beside them, + # as the Debian workflow keeps its own. Still in the workspace where + # the action wrote it. + - name: Upload source package to S3 + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} - path: kernel-build/${{ inputs.distro }} - destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.distro }}/${{ github.run_id }}-${{ github.run_attempt }}/ + path: source-package + destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/source/ diff --git a/.github/workflows/build-kernel.yml b/.github/workflows/build-kernel.yml deleted file mode 100644 index aa37b234..00000000 --- a/.github/workflows/build-kernel.yml +++ /dev/null @@ -1,283 +0,0 @@ -name: build-kernel - -# Controls when the workflow will run -on: - workflow_dispatch: - inputs: - qcom-build-utils-ref: - description: qcom-build-utils ref branch or commit - required: false - default: 'main' - kernel-branch: - description: Kernel Branch or Tag to sync - required: false - default: 'qcom-next' - kernel-url: - description: Custom Kernel Repo URL (Leave empty for default qualcomm-linux/kernel) - required: false - default: '' - qcom-next-pr: - description: Space-separated PR numbers to merge from qcom-next - type: string - required: false - default: '' - kernel-topics-pr: - description: Space-separated list of kernel-topics PR numbers - type: string - required: false - default: '' - - secrets: - DEB_PKG_BOT_CI_TOKEN: - required: true - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - - env: - KERNEL_BRANCH: ${{ github.event.inputs.kernel-branch || 'qcom-next' }} - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 - - - name: Extract org, repo and branch name - run: | - # Extract org (owner) and repo name from GITHUB_REPOSITORY (format: org/repo) - echo "ORG_NAME=${GITHUB_REPOSITORY%%/*}" >> "$GITHUB_ENV" - echo "REPO_NAME=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_ENV" - - - name: Debug print exported env - run: | - echo "ORG_NAME=$ORG_NAME" - echo "REPO_NAME=$REPO_NAME" - - - name: Checkout qcom-build-utils - uses: actions/checkout@v4 - with: - repository: qualcomm-linux/qcom-build-utils - ref: ${{ inputs.qcom-build-utils-ref }} - token: ${{ secrets.DEB_PKG_BOT_CI_TOKEN }} - path: ./qcom-build-utils - fetch-depth: 1 - - - name: Deduce qcom-build-utils head commit SHA - run: | - # Get the tip commit SHA of the qcom-build-utils branch/tag used - cd qcom-build-utils - QCOM_BUILD_UTILS_SHA=$(git rev-parse HEAD) - echo "QCOM_BUILD_UTILS_SHA=$QCOM_BUILD_UTILS_SHA" >> "$GITHUB_ENV" - # Echo for logs - echo "Head SHA for qcom-build-utils: $QCOM_BUILD_UTILS_SHA" - - - name: Pull docker image - run: | - docker pull artifacts.codelinaro.org/clo-420-qli-registry/kmake-image-ubuntu-noble-arm64:ver.1.0 - docker tag artifacts.codelinaro.org/clo-420-qli-registry/kmake-image-ubuntu-noble-arm64:ver.1.0 kmake-image:ubuntu-noble-arm64 - - # Runs if kernel-url is EMPTY - - name: Sync kernel codebase (Default) - if: ${{ inputs.kernel-url == '' }} - env: - PAT: ${{ secrets.PAT }} - run: | - cd qcom-build-utils - cd kernel && export BUILD_TOP=`pwd` - echo "build_top=$BUILD_TOP" >> "$GITHUB_ENV" - - export REPO="qualcomm-linux/kernel" - echo "kernel_repo=$REPO" >> "$GITHUB_ENV" - echo "Syncing ${REPO} ref ${KERNEL_BRANCH}" - - git ls-remote --exit-code --heads "https://${PAT}@github.com/${REPO}.git" ${KERNEL_BRANCH} && IS_BRANCH=true || IS_BRANCH=false - - git init $BUILD_TOP/qcom-kernel - cd $BUILD_TOP/qcom-kernel - git remote add origin "https://${PAT}@github.com/${REPO}.git" - - if [ "$IS_BRANCH" = true ]; then - echo "Detected branch: ${KERNEL_BRANCH}" - git fetch --depth=1 origin ${KERNEL_BRANCH} - git checkout ${KERNEL_BRANCH} - else - echo "Assuming tag: ${KERNEL_BRANCH}" - git fetch --depth=1 origin tag ${KERNEL_BRANCH} - git checkout FETCH_HEAD - fi - - # Get Head SHA of kernel branch/tag cloned - QCOM_KERNEL_SHA=$(git rev-parse HEAD) - echo "QCOM_KERNEL_SHA=$QCOM_KERNEL_SHA" >> "$GITHUB_ENV" - - # Runs if kernel-url is PROVIDED - - name: Sync custom kernel codebase - if: ${{ inputs.kernel-url != '' }} - env: - KERNEL_URL_INPUT: ${{ inputs.kernel-url }} - run: | - cd qcom-build-utils - mkdir -p kernel - cd kernel && export BUILD_TOP=`pwd` - echo "build_top=$BUILD_TOP" >> "$GITHUB_ENV" - - KERNEL_URL="$KERNEL_URL_INPUT" - echo "Syncing custom repo: ${KERNEL_URL} ref ${KERNEL_BRANCH}" - - git ls-remote --exit-code --heads ${KERNEL_URL} ${KERNEL_BRANCH} && IS_BRANCH=true || IS_BRANCH=false - - mkdir -p $BUILD_TOP/qcom-kernel - cd $BUILD_TOP/qcom-kernel - git init - git remote add origin ${KERNEL_URL} - - if [ "$IS_BRANCH" = true ]; then - echo "Detected branch: ${KERNEL_BRANCH}" - git fetch --depth=1 origin ${KERNEL_BRANCH} - git checkout ${KERNEL_BRANCH} - else - echo "Assuming tag: ${KERNEL_BRANCH}" - git fetch --depth=1 origin tag ${KERNEL_BRANCH} - git checkout FETCH_HEAD - fi - - QCOM_KERNEL_SHA=$(git rev-parse HEAD) - echo "QCOM_KERNEL_SHA=$QCOM_KERNEL_SHA" >> "$GITHUB_ENV" - - - name: Merge qcom-next PR patches - if: ${{ inputs.kernel-url == '' }} - env: - QCOM_NEXT_PR_INPUT: ${{ github.event.inputs.qcom-next-pr }} - PAT: ${{ secrets.PAT }} - run: | - cd "$build_top/" - if [ -z "$QCOM_NEXT_PR_INPUT" ]; then - echo "No PR number specified, skipping merge." - else - echo "Merging PR(s) #$QCOM_NEXT_PR_INPUT" - - cd "$build_top/qcom-kernel" - - for pr in $QCOM_NEXT_PR_INPUT; do - echo "::group::Merging qcom-next PR #$pr" - git fetch "https://${PAT}@github.com/${kernel_repo}.git" "pull/$pr/head:pr-$pr" - - if ! git merge "pr-$pr" --no-commit; then - echo "Merge conflict while merging PR #$pr. Aborting merge and failing the job." - git merge --abort || true - exit 1 - fi - - if ! git diff --cached --quiet; then - git commit -m "Merged PR #$pr" - else - echo "Nothing to commit for PR #$pr (already merged or fast-forwarded)." - git merge --abort 2>/dev/null || true - fi - echo "::endgroup::" - done - fi - - - name: Apply kernel-topics PR patches - if: ${{ inputs.kernel-url == '' }} - env: - KERNEL_TOPICS_PR_INPUT: ${{ github.event.inputs.kernel-topics-pr }} - run: | - cd "$build_top/" - if [ -z "$KERNEL_TOPICS_PR_INPUT" ]; then - echo "Nothing to apply in kernel topics." - else - echo "Applying PR(s) from topics branch #$KERNEL_TOPICS_PR_INPUT" - for pr in $KERNEL_TOPICS_PR_INPUT; do - echo "::group::Applying kernel-topics PR #$pr" - wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - - - cd "$build_top/qcom-kernel" - - if ! git am "../$pr.patch"; then - echo "Patch application failed for PR #$pr. Aborting 'git am' and failing the job." - git am --abort || true - exit 1 - fi - echo "::endgroup::" - cd "$build_top/" - done - fi - - - name: Enable Kernel Configs - run: | - cd qcom-build-utils/kernel - export BUILD_TOP=$(pwd) - ./scripts/enable_squashfs_configs.sh $BUILD_TOP/qcom-kernel/ - - - name: Run build script - run: | - docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" kmake-image:ubuntu-noble-arm64 \ - -c ' - cd qcom-build-utils/kernel - export BUILD_TOP=`pwd` - ./scripts/build_kernel.sh $BUILD_TOP/qcom-kernel/ - ' - - - name: Build Kernel Debian Package - env: - JOB_ID: ${{ github.run_id }} - JOB_ATTEMPT: ${{ github.run_attempt }} - run: | - docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" \ - -e JOB_ID -e JOB_ATTEMPT \ - kmake-image:ubuntu-noble-arm64 \ - -c ' - cd qcom-build-utils/kernel - export BUILD_TOP=`pwd` - ./scripts/build-kernel-deb.sh out/ ${JOB_ID}-${JOB_ATTEMPT} - mkdir -p deb_artifact - cp ./*.deb deb_artifact/ - ' - - - name: Generate build_info metadata - env: - JOB_ID: ${{ github.run_id }} - JOB_ATTEMPT: ${{ github.run_attempt }} - QCOM_BUILD_UTILS_REF_INPUT: ${{ inputs.qcom-build-utils-ref }} - KERNEL_URL_INPUT: ${{ inputs.kernel-url }} - KERNEL_BRANCH_INPUT: ${{ inputs.kernel-branch }} - QCOM_NEXT_PR_INPUT: ${{ inputs.qcom-next-pr }} - KERNEL_TOPICS_PR_INPUT: ${{ inputs.kernel-topics-pr }} - run: | - docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" \ - -e JOB_ID -e JOB_ATTEMPT -e ORG_NAME -e REPO_NAME \ - -e QCOM_BUILD_UTILS_REF_INPUT -e QCOM_BUILD_UTILS_SHA \ - -e KERNEL_URL_INPUT -e KERNEL_BRANCH_INPUT -e QCOM_KERNEL_SHA \ - -e QCOM_NEXT_PR_INPUT -e KERNEL_TOPICS_PR_INPUT \ - kmake-image:ubuntu-noble-arm64 \ - -c ' - cd qcom-build-utils/kernel - { - echo "JOB_ID: $JOB_ID" - echo "JOB_ATTEMPT: $JOB_ATTEMPT" - echo "ORG_NAME: $ORG_NAME" - echo "REPO_NAME: $REPO_NAME" - echo "QCOM-BUILD-UTILS BRANCH/TAG: $QCOM_BUILD_UTILS_REF_INPUT" - echo "QCOM-BUILD-UTILS HEAD SHA: $QCOM_BUILD_UTILS_SHA" - echo "CUSTOM KERNEL URL: $KERNEL_URL_INPUT" - echo "KERNEL BRANCH/TAG: $KERNEL_BRANCH_INPUT" - echo "KERNEL HEAD SHA: $QCOM_KERNEL_SHA" - echo "PRs FROM QCOM-NEXT: $QCOM_NEXT_PR_INPUT" - echo "PRs FROM KERNEL TOPICS: $KERNEL_TOPICS_PR_INPUT" - } > deb_artifact/build_info - ' - - - name: Upload kernel .deb package to S3 - uses: qualcomm-linux/upload-private-artifact-action@aws - with: - s3_bucket: qli-prd-lecore-gh-artifacts - path: qcom-build-utils/kernel/deb_artifact - destination: ${{ env.ORG_NAME }}/pkg/temp/${{ env.REPO_NAME }}/${{ github.run_id }}-${{ github.run_attempt }}/ diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index a3022f7e..6ee6bab4 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,20 +1,30 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel -# build per Daily entry in ci/build-matrix.json after suite flattening. -# Manual dispatch can run the full Daily matrix, every suite for one kernel -# variant, or one kernel variant and suite. Variant and suite are matrix values, -# so future variants need no workflow edit. +# build per entry in ci/build-matrix.yaml. # -# Matrix schema: ci/build-matrix.json -# Each row with "type": "Daily" is expanded by ci/scripts/resolve-matrix.sh: -# the "suites" array is flattened into one job per suite. All other fields -# (kernel_variant, srcpkg, binpkg, kernel_config, dkms, debian_revision, -# git_clone, branch_or_tag, ref_strategy) are passed through to -# build-kernel-deb.yml as inputs. +# This is also the manual build entry point. Everything about a build comes +# from its matrix entry, so a dispatch only says which entries to run: "all", +# or a comma-separated list of build names. Build names are matrix values, so +# a new one needs no workflow edit. # -# To add a daily build target: add a row to ci/build-matrix.json with -# "type": "Daily". No workflow changes needed. +# This is the only workflow that publishes a kernel package. Its Debian +# entries are promoted into the target workspace as part of the build that +# produced them, so what the archive carries is always something this workflow +# built and tested. Nothing promotes afterwards: the CI workspace a build runs +# in is named after that run and does not outlive it. +# +# Matrix schema: ci/build-matrix.yaml +# One entry is one generated package, so ci/scripts/resolve-matrix.py passes +# the selected entries through as they stand. Every field (name, suite, +# flavour, srcpkg, binpkg, kernel_config, dkms, debian_revision, git_clone, +# branch_or_tag, ref_strategy) becomes an input of the build workflow for the +# entry's family. name labels the job; flavour is what reaches the built +# kernel's version and its published paths. Where the result is published is +# this workflow's to decide, not the entry's. +# +# To add a daily build target: add an entry to ci/build-matrix.yaml. No +# workflow changes needed. on: schedule: @@ -22,24 +32,10 @@ on: - cron: '0 23 * * *' workflow_dispatch: inputs: - build-scope: - description: 'Choose all variants, all suites for one variant, or one variant and suite' - type: choice + builds: + description: 'Builds to run: all, or comma-separated build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-debug-forky)' required: true - default: 'Selected variant and suite' - options: - - 'Full matrix' - - 'Selected variant (all suites)' - - 'Selected variant and suite' - kernel-variant: - description: 'Kernel variant to build (ignored for Full matrix)' - required: true - default: 'qcom-next' - type: string - suite: - description: 'Target suite to build (used only for Selected variant and suite)' - required: true - default: 'trixie' + default: 'all' type: string permissions: @@ -48,78 +44,97 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Daily rows from ci/build-matrix.json. + # configure-matrix: select the entries to build from ci/build-matrix.yaml. # - # Delegates to ci/scripts/resolve-matrix.sh which filters type=Daily rows - # and expands each row's "suites" array into one flat entry per suite. - # Manual dispatch selects the complete matrix, every suite for a variant, or - # one variant and suite. + # Delegates to ci/scripts/resolve-matrix.py, which validates the whole matrix + # and returns its entries. A dispatch narrows that to a comma-separated list + # of build names; "all", and a scheduled run, take the whole matrix. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + debian: ${{ steps.set-matrix.outputs.debian }} + ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Generate build matrix id: set-matrix env: - BUILD_SCOPE_INPUT: ${{ github.event.inputs.build-scope }} - INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} - INPUT_SUITE: ${{ github.event.inputs.suite }} + INPUT_BUILDS: ${{ inputs.builds }} run: | set -euo pipefail - BUILD_SCOPE="$BUILD_SCOPE_INPUT" + # A scheduled run carries no inputs, so it lands on the dispatch + # default: the whole matrix. + BUILDS="${INPUT_BUILDS:-all}" - if [[ "${{ github.event_name }}" == "schedule" ]]; then - BUILD_SCOPE="Full matrix" + args=() + if [[ "$BUILDS" == "all" ]]; then + echo "Source: ci/build-matrix.yaml (every entry)" + else + echo "Source: ci/build-matrix.yaml (builds: $BUILDS)" + # A name matching no entry fails here, so a typo stops the run + # rather than quietly narrowing it to nothing. + args+=(--build "$BUILDS") fi - case "$BUILD_SCOPE" in - "Full matrix") - echo "Source: ci/build-matrix.json (full Daily matrix)" - args=(--type Daily) - ;; - "Selected variant (all suites)") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" - args=(--type Daily --kernel-variant "$INPUT_VARIANT") - ;; - "Selected variant and suite") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Daily --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") - ;; - *) - echo "ERROR: unsupported build scope: $BUILD_SCOPE" >&2 - exit 1 - ;; - esac - MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") - - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" + # One selection, split by build path. The two families are built by + # different workflows, so each gets the entries it builds and neither + # run starts a job for the other. A narrowed dispatch can leave one + # side empty, which --allow-empty reports as [] rather than as a + # failure; a name that matches nothing still fails, in both calls. + for family in debian ubuntu; do + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}" \ + --family "$family" --allow-empty) + echo "$family=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$family entries: $(echo "$MATRIX" | jq length)" + done # --------------------------------------------------------------------------- - # build: one parallel job per flat matrix entry. + # build-debian / build-ubuntu: one parallel job per matrix entry. + # + # Two jobs rather than one because the families are built by different + # workflows, on different runners, and published from different places. + # Splitting the matrix by family means each entry starts exactly the jobs + # that build it: a run shows no skipped job for the path it did not take. # - # Each entry calls build-kernel-deb.yml as a reusable workflow. The full - # set of matrix fields (suite, srcpkg, binpkg, kernel_config, dkms, etc.) is - # threaded through as inputs. fail-fast is disabled so a failed trixie - # build does not cancel the forky build. + # Both are named ${{ matrix.name }}, so the checks list reads the same as + # one job would -- the split is in what each entry calls, not in what it is + # called. fail-fast is disabled so a failed trixie build does not cancel the + # forky build. + # + # The if: guards catch a family the run selects nothing on -- a narrowed + # dispatch, or a matrix with no entries for it. They are not optional: an + # empty matrix is a workflow error rather than an empty job list. + # + # A skipped family shows up as a check literally named "matrix.name": GitHub + # never expands a job it skips, so it prints the expression's source text, + # and a || fallback prints just the same. Only a name that never mentions + # matrix would read well, which would mean renaming every check in every + # workflow to tidy up one that appears when a family is empty. So it stands. # --------------------------------------------------------------------------- - build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + build-debian: + name: ${{ matrix.name }} needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.debian != '[]' }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} @@ -132,7 +147,51 @@ jobs: localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + # Released archives only. This build is the one a release promotes, so + # anything it resolves a Build-Depends against is something the released + # kernel will depend on: reading qli-staging here would let a kernel + # reach qli depending on a -dkms package that has not. + extra-build-dep-workspaces: qli + # Every Debian entry the nightly builds is promoted into this workspace + # once its build succeeds. The destination belongs to the run rather than + # to the entry: this is the nightly, so its results are what the archive + # is for. A PR build of the same entry passes nothing here and promotes + # nowhere. + # + # qli-staging rather than qli: a nightly promotion is unattended, so it + # publishes where an unreviewed kernel belongs. Pointing this at qli -- + # by setting the repository variable -- would put every night's build + # straight into the released archive, so move the promote job to the + # Production environment first, and take its approval gate with it. + target-workspace: ${{ vars.DEBUSINE_STAGING_WORKSPACE || 'qli-staging' }} secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + + build-ubuntu: + name: ${{ matrix.name }} + needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.ubuntu != '[]' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-ubuntu.yml + with: + suite: ${{ matrix.suite }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + dkms: ${{ matrix.dkms }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: ${{ matrix.kver_extra || '' }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 4f9e5e3a..8f08cee2 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -1,6 +1,10 @@ name: PR Build # Build-test PRs through the same pipeline the daily builds use. +# +# Packaging and CI now live on one branch, so a PR is tested entirely against +# its own commit: its ci/build-matrix.yaml defines the legs, and its debian/ +# tree and ci/scripts build them. Nothing is read from main. on: pull_request: @@ -16,7 +20,7 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Daily rows from the PR's build-matrix.json. + # configure-matrix: select the entries from the PR's build-matrix.yaml. # # The PR head, not the merge commit: a push mid-run must not change what this # run tested. A SHA also resolves for fork PRs, whose head branch does not @@ -25,10 +29,13 @@ jobs: configure-matrix: name: Configure build matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + debian: ${{ steps.set-matrix.outputs.debian }} + ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false @@ -37,27 +44,44 @@ jobs: id: set-matrix run: | set -euo pipefail - MATRIX=$(ci/scripts/resolve-matrix.sh --type Daily) - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" + # Split by build path: the families are built by different workflows, + # so a PR gets one job per leg it actually builds and no skipped job + # for the path that leg does not take. + for family in debian ubuntu; do + MATRIX=$(ci/scripts/resolve-matrix.py \ + --family "$family" --allow-empty) + echo "$family=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$family entries: $(echo "$MATRIX" | jq length)" + done # --------------------------------------------------------------------------- - # build: one parallel job per flat matrix entry, mirroring daily.yml. + # build-debian / build-ubuntu: one parallel job per matrix entry, mirroring + # daily.yml. + # + # A PR builds the whole matrix, so it gets one check per build leg and + # nothing else -- unless the matrix has no entries for one family, which + # fires that family's guard and leaves a check named "matrix.name". + # daily.yml documents why the guard is needed and the name left alone. # # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. # --------------------------------------------------------------------------- - build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + build-debian: + name: ${{ matrix.name }} needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.debian != '[]' }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} @@ -68,11 +92,60 @@ jobs: dkms: ${{ matrix.dkms }} debian-revision: ${{ matrix.debian_revision }} localversion: ${{ matrix.localversion || '' }} - # Mark the version so PR artifacts never collide with the daily build's - # in the shared Debusine workspace. + # Marks the kernel release, not the package version: debian/rules + # appends KVER_EXTRA to KVER and then derives the version separately, so + # this changes uname -r and the versioned linux-image name while the + # source version stays the one the nightly would build. That is enough + # here -- a PR build has its own throwaway workspace and promotes + # nowhere -- but it is not a version stamp, and nothing should start + # relying on it as one. kver-extra: -pr${{ github.event.number }} + # Staging as well as released: nothing here is promoted anywhere, so a + # pull request may depend on a -dkms package that has reached qli-staging + # and not yet qli -- which is how a kernel and the module it needs get + # reviewed together. + extra-build-dep-workspaces: qli qli-staging + # Empty, and stated rather than left to the input's default: this is the + # one value keeping a pull request's kernel out of the archives. An empty + # target-workspace is what build-kernel-debian.yml's promote job tests + # for, so a PR is built, tested and published to S3 and goes no further. + # + # A PR is unreviewed by definition, and its packages are versioned + # -pr rather than as the nightly's, so promoting one would put a + # kernel nobody has approved into an archive people install from, under a + # version no release will ever supersede. Left implicit, a future edit to + # the default -- or a copy of this block into a workflow that should + # promote -- would flip that silently; spelled out here, changing it + # means saying so. + target-workspace: '' secrets: - # No DEBUSINE_RELEASE_TOKEN: PR builds never promote to a release - # workspace, so they leave target-workspace empty. DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + + build-ubuntu: + name: ${{ matrix.name }} + needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.ubuntu != '[]' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-ubuntu.yml + with: + suite: ${{ matrix.suite }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + dkms: ${{ matrix.dkms }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: -pr${{ github.event.number }} diff --git a/.github/workflows/qcom-preflight-checks.yml b/.github/workflows/qcom-preflight-checks.yml new file mode 100644 index 00000000..51b7aca4 --- /dev/null +++ b/.github/workflows/qcom-preflight-checks.yml @@ -0,0 +1,24 @@ +name: Qualcomm Preflight Checks +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + security-events: write + +jobs: + preflight: + name: Run QC Preflight Checks + uses: qualcomm/qcom-reusable-workflows/.github/workflows/reusable-qcom-preflight-checks-orchestrator.yml@v2 + with: + enable-semgrep-scan: true + enable-dependency-review: true + enable-repolinter-check: true + enable-copyright-license-check: true + enable-commit-email-check: true + enable-commit-msg-check: false + enable-armor-checkers: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32eddcf5..0e04d5c9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,39 +1,37 @@ name: release -# Release kernel package builds. Triggered manually only. +# Release kernel packages into the released archive. # -# Reads Release rows from ci/build-matrix.json, flattens suites, and builds -# each entry using the matrix-selected ref strategy and branch_or_tag. After a -# successful build, packages are promoted from the ephemeral Debusine CI -# workspace to the stable target workspace (e.g. qli) via debusine-action -# lib/release. +# A release builds the ref it releases. It cannot promote a nightly instead: +# lib/build names the CI workspace after the run that created it and that +# workspace does not outlive the run, so by the time anyone decides to release, +# there is no longer anything to promote from. Building here means the artifact +# that reaches the archive is the one this run produced and tested, and the +# ref it came from is pinned rather than resolved, so two dispatches of one +# entry release the same kernel. # -# To update a release ref: open a PR that changes the Release row for that -# kernel variant in ci/build-matrix.json. +# Every value describing a release lives in ci/build-matrix.yaml under +# "releases": the ref, the suites, the packaging, and the workspace each entry +# publishes into. A dispatch only says which of them to run. Updating a release +# is therefore a pull request that changes branch_or_tag, which is what puts +# the ref that ships under review -- rather than a version typed into this form +# at the moment of releasing. # -# This workflow intentionally has no schedule trigger. Releases are always -# initiated manually after validation. +# Debian suites only. Promotion runs through Debusine, and the Ubuntu path +# publishes to S3 with no workspace to promote into; resolve-matrix.py rejects +# a non-Debian release entry, so there is no Ubuntu leg to select here. +# +# Triggered manually only, and never on a schedule: a release is a decision. +# The promote job runs in the Production environment, so whatever approval that +# environment requires stands between this run and the archive. on: workflow_dispatch: inputs: - release-scope: - description: 'Choose all suites for one variant or one variant and suite' - type: choice - required: true - default: 'Selected variant (all suites)' - options: - - 'Selected variant (all suites)' - - 'Selected variant and suite' - kernel-variant: - description: 'Kernel variant to release' - required: true - default: 'qcom-next' - type: string - suite: - description: 'Target suite to release (used only for Selected variant and suite)' + builds: + description: 'Releases to run: all, or comma-separated build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-forky)' required: true - default: 'trixie' + default: 'all' type: string permissions: @@ -42,70 +40,98 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Release rows from ci/build-matrix.json. + # configure-matrix: select the entries to release from ci/build-matrix.yaml. + # + # --releases reads the releases list rather than builds. Both are validated + # whichever is selected, so a broken release entry fails here rather than + # part-way through a release. + # + # No --allow-empty and no family split: every release entry is Debian, so a + # selection that matches nothing is a mistake in the request and fails. # --------------------------------------------------------------------------- configure-matrix: name: Configure release matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + releases: ${{ steps.set-matrix.outputs.releases }} steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Generate release matrix id: set-matrix env: - RELEASE_SCOPE_INPUT: ${{ github.event.inputs.release-scope }} - INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} - INPUT_SUITE: ${{ github.event.inputs.suite }} + INPUT_BUILDS: ${{ inputs.builds }} run: | set -euo pipefail - RELEASE_SCOPE="$RELEASE_SCOPE_INPUT" + BUILDS="${INPUT_BUILDS:-all}" - case "$RELEASE_SCOPE" in - "Selected variant (all suites)") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" - args=(--type Release --kernel-variant "$INPUT_VARIANT") - ;; - "Selected variant and suite") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Release --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") - ;; - *) - echo "ERROR: unsupported release scope: $RELEASE_SCOPE" >&2 - exit 1 - ;; - esac - MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") + args=() + if [[ "$BUILDS" == "all" ]]; then + echo "Source: ci/build-matrix.yaml releases (every entry)" + else + echo "Source: ci/build-matrix.yaml releases (builds: $BUILDS)" + # A name matching no release entry fails here, so releasing under a + # stale or mistyped name stops the run rather than quietly + # releasing a subset of what was asked for. + args+=(--build "$BUILDS") + fi - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" + MATRIX=$(ci/scripts/resolve-matrix.py --releases "${args[@]}") + + echo "releases=$MATRIX" >> "$GITHUB_OUTPUT" + echo "Release entries: $(echo "$MATRIX" | jq length)" + + - name: Summarise what this run will release + env: + MATRIX: ${{ steps.set-matrix.outputs.releases }} + run: | + set -euo pipefail + # The refs and destinations in one place, so the approval on the + # Production environment is given against what is actually shipping + # rather than against a run number. + { + echo "### Releasing" + echo + echo "| Build | Suite | Ref | Revision | Into |" + echo "| --- | --- | --- | --- | --- |" + jq -r '.[] | "| \(.name) | \(.suite) | `\(.branch_or_tag)` | `\(.debian_revision)` | \(.target_workspace) |"' <<<"$MATRIX" + } >> "$GITHUB_STEP_SUMMARY" # --------------------------------------------------------------------------- - # build: one parallel job per flat Release matrix entry. + # release: one parallel job per selected entry. + # + # The same workflow the nightly calls, differing only in what it is told: a + # pinned ref instead of a resolved one, and the released workspace instead of + # staging. Nothing about how a kernel is built changes because it is being + # released. # - # Uses the ref strategy and branch_or_tag selected by the matrix. - # Passes target_workspace so build-kernel-debusine.yml triggers the Debusine - # release path (lib/release) instead of the S3 publish path. + # fail-fast is disabled so a failed trixie release does not cancel the forky + # one; each entry is a package of its own and one failing does not make the + # others wrong to ship. # --------------------------------------------------------------------------- - build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + release: + name: ${{ matrix.name }} needs: configure-matrix strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + include: ${{ fromJson(needs.configure-matrix.outputs.releases) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} - tag-pattern: ${{ matrix.tag_pattern || '' }} srcpkg: ${{ matrix.srcpkg }} binpkg: ${{ matrix.binpkg }} kernel-config: ${{ matrix.kernel_config }} @@ -114,8 +140,20 @@ jobs: localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + # Released archives only, as the nightly does. What this build resolves a + # Build-Depends against is something the released kernel will depend on, + # so reading qli-staging here would let a released kernel require a -dkms + # package that has not been released. + extra-build-dep-workspaces: qli + # The entry's own, and the one field that makes it a release rather than + # a build of the same ref: where the package is published. Stated in the + # matrix rather than defaulted here, so the archive a release reaches is + # reviewable alongside the ref it ships. target-workspace: ${{ matrix.target_workspace }} + # A release reaches an archive people install from, so the promotion + # waits for whatever approval the Production environment requires. The + # nightly leaves this at Staging and promotes unattended. + promote-environment: Production secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} diff --git a/.github/workflows/stale-issues.yaml b/.github/workflows/stale-issues.yaml index e84b54b4..07530ad4 100644 --- a/.github/workflows/stale-issues.yaml +++ b/.github/workflows/stale-issues.yaml @@ -11,7 +11,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: stale-issue-message: 'This issue has been marked as stale due to 60 days of inactivity. To prevent automatic closure in 10 days, remove the stale label or add a comment. You can reopen a closed issue at any time.' stale-pr-message: 'This pull request has been marked as stale due to 60 days of inactivity. To prevent automatic closure in 10 days, remove the stale label or add a comment. You can reopen a closed pull request at any time.' diff --git a/.gitignore b/.gitignore index 2583b54a..d2fc2efd 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ debian/linux-image-*.prerm # Default kernel source and build output directories kernel-source/ kernel-build/ + +# Python bytecode cache from ci/scripts/ +__pycache__/ diff --git a/README.md b/README.md index 1c60c355..ac918fd0 100644 --- a/README.md +++ b/README.md @@ -13,233 +13,320 @@ takes its packaging and its CI scripts from the same commit. The CI model is matrix-driven. This single repository can deliver multiple kernel variants, each with independent source/package identity, kernel source and ref strategy, configuration fragments, Debian revision, target suites, and -release destination. +archive destination. -Every kernel variant owns exactly two complete matrix rows: one `Daily` row and -one `Release` row. The resolver expands every suite in those rows into an -isolated `kernel_variant + suite` build leg. +Every entry in the matrix is one isolated build leg producing one package. A +kernel flavour owns as many entries as it has suites. ### Configured variants -| Variant | Source package | Image metapackage | Daily suites | Release suites | Notes | -|---------|----------------|-------------------|--------------|----------------|-------| -| `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | trixie, forky | Standard kernel | -| `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | trixie, forky | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | - -Both build the same kernel ref. `derive-localversion.sh` folds the variant name -into LOCALVERSION, so each produces a distinct kernel release -(`-qcom-next-` and `-qcom-next-debug-`) and therefore a distinct -versioned image package that can be installed alongside the other. - -`ci/build-matrix.json` is the source of truth; this table is a summary. - -Two entry points use the same reusable build pipeline: - -- **Daily** uses the matrix-selected latest-tag or branch-tip strategy and - builds every configured Daily suite. -- **Release** uses a pinned matrix ref and promotes successful Debian packages - to the selected production Debusine workspace. - -The final Production matrix is conceptually: - -```json -{ - "suite_suffix_mapping": { - "trixie": "~bpo13+1", - "forky": "", - "resolute": "~26.04.1" - }, - "deliveries": [ - { - "kernel_variant": "qcom-next", - "type": "Daily", - "suites": ["trixie", "forky", "resolute"], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "qcom-next", - "ref_strategy": "latest_tag", - "tag_pattern": "qcom-next-*", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli", - "debian_version_suffix": "~" - }, - { - "kernel_variant": "qcom-next", - "type": "Release", - "suites": ["trixie", "forky"], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "", - "ref_strategy": "pinned_ref", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli", - "debian_version_suffix": "", - "target_workspace": "qli" - } - ] -} +| Variant | Source package | Image metapackage | Suites | Notes | +|---------|----------------|-------------------|--------|-------| +| `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | Standard kernel | +| `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | +| `qcom-arduino` | `linux-qcom-arduino` | `linux-image-qcom-arduino` | trixie, forky | Arduino hardware-enablement topic branch (`early/hwe/arduino` of `kernel-topics`) | +| `mainline` | `linux-mainline` | `linux-image-mainline` | trixie, forky | Tip of Linus's tree, tracked for early warning of upstream breakage. No DKMS modules | +| `next` | `linux-next` | `linux-image-next` | trixie, forky | Newest `next-YYYYMMDD` tag of linux-next. No DKMS modules | + +`derive-localversion.sh` folds the *flavour* into LOCALVERSION, so each +produces a distinct kernel release (`+qcom-next--g`, +`+qcom-next-debug--g`, and so on) and therefore a distinct versioned +image package that can be installed alongside the others. The flavour is what +the kernel is; a build's `name` is only what CI calls it. See +[docs/version.md](docs/version.md) for how the version strings are composed. + +The last three track a moving upstream for early warning. They are built and +promoted like the rest; nothing about an entry says where it goes. + +`ci/build-matrix.yaml` is the source of truth; this table is a summary. + +Entries come in two lists, each with a workflow that builds them: + +- **`daily.yml`** builds every `builds` entry nightly, using the matrix-selected + latest-tag or branch-tip strategy, and promotes its Debian entries into the + staging archive as part of the build that produced them. +- **`release.yml`** builds a `releases` entry on request, from the immutable ref + that entry pins, and promotes it into `qli` behind an approval. + +Neither publishes a build some earlier run produced. The Debusine workspace a +build runs in is named after that run and does not outlive it, so the only +moment its contents can be published is while the run still holds it — +promotion is in the build or it is nowhere, and a release therefore builds the +ref it ships rather than promoting a nightly. What the archive carries is +therefore always an artifact some nightly run built and tested. + +One entry in `builds` is one generated package: a single `name` for a single +`suite`. The `name` labels that one build and nothing else — it is the Actions +job name and what a manual dispatch asks for. Nothing is expanded or derived at +resolve time, so what an entry says is what gets built: + +```yaml +builds: + - name: qcom-next-trixie + suite: trixie + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' ``` -`suite_suffix_mapping` is matrix-wide policy, not duplicated per row: every -suite referenced by any row's `suites` must have an entry here, and every -delivery for a variant derives its final `debian_revision` as -`debian_version_stub + suite_suffix_mapping[suite] + delivery_suffix`, where -`delivery_suffix` is `~` for Daily and empty for Release. For the values -above: +Entries are written out in full rather than sharing YAML anchors, so each one +can be read, grepped, and changed on its own. `resolve-matrix.py` enforces the +consistency that duplication would otherwise put at risk — see +[Matrix Model](#matrix-model). -| Suite | Daily | Release | -| --- | --- | --- | -| Trixie | `0qli~bpo13+1~` | `0qli~bpo13+1` | -| Forky | `0qli~` | `0qli` | -| Resolute | `0qli~26.04.1~` | (not a configured Release suite) | +Each entry states its `debian_revision` outright. The configured values are: + +| Suite | Revision | +| --- | --- | +| Trixie | `0qli1~bpo13+1` | +| Forky | `0qli1` | +| Resolute | `0qli1~26.04.1` | + +None of them carries a trailing `~`. That marker existed to sort a daily build +below the release rebuilt from the same ref, and there is no such rebuild: the +version is decided once, when the package is built, and the artifact any +archive holds is that one. With nothing for it to sort against, a trailing `~` +would only make every published kernel look permanently provisional. -`~` always sorts below the same prefix without it in Debian version -ordering, so Daily always sorts below Release for the same suite and stub. Ordering across *different* suites depends entirely on the configured -suffixes: with the mapping above, Resolute < Trixie < Forky for the same -delivery type, matching a Debian-backports-then-unstable promotion chain. -This is a deliberate ordering policy, not an automatic guarantee — adding a -suite means choosing a suffix that sorts where that suite belongs relative to -the others. One nuance to be aware of: because Forky's suffix is empty, its -Daily revision ends immediately after the trailing `~`, so Trixie Daily does -not sort below Forky Daily even though Trixie Release sorts below Forky -Release. This does not affect the supported Release-to-Release upgrade path. - -`ci/build-matrix.json` is the authoritative configuration. Adding a kernel -variant is a two-row matrix change, not a workflow redesign. +revisions: with the values above, Resolute < Trixie < Forky, matching a +Debian-backports-then-unstable promotion chain. This is a deliberate ordering +policy, not an automatic guarantee — adding a suite means choosing a revision +that sorts where that suite belongs relative to the others. + +`ci/build-matrix.yaml` is the authoritative configuration. Adding a kernel +variant is a matrix change, not a workflow redesign. ## Workflows | Workflow | Purpose | Trigger | | --- | --- | --- | -| `daily.yml` | Resolves and runs the Daily matrix. | Scheduled daily at `23:00 UTC`, or manual dispatch. | -| `release.yml` | Resolves and runs the Release matrix. | Manual dispatch only. | -| `build-kernel-deb.yml` | Reusable orchestrator for one kernel variant and suite. | Manual dispatch or called by Daily and Release. | -| `build-kernel-debusine.yml` | Builds Debian suites in Debusine and either publishes Daily artifacts or promotes Releases. | Called by `build-kernel-deb.yml`. | -| `build-kernel-ubuntu.yml` | Builds Ubuntu-family suites with the Docker path. | Called by `build-kernel-deb.yml`. | +| `daily.yml` | Resolves and runs the matrix. The manual build entry point. | Scheduled daily at `23:00 UTC`, or manual dispatch. | +| `release.yml` | Builds the pinned refs in the matrix's `releases` list and promotes them into the released archive. | Manual dispatch only. | +| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine, publishes it to S3, and promotes it into the workspace its caller named, if any. | Called by Daily, Release and PR build. | +| `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Called by Daily and PR build. | + +The two build workflows share their steps through two composite actions rather +than through a common orchestrator workflow: + +| Action | Used by | +| --- | --- | +| `.github/actions/prepare-kernel-source` | Both, as the `prepare` job. | +| `.github/actions/debusine-build` | The Debian workflow, as the `build` job. | + +Which of them a build leg calls is decided by the caller, from the entry's +suite: `resolve-matrix.py --family debian|ubuntu` splits the selection, and +each family's entries call only the workflow that builds them. Nothing inside a +build workflow is conditional on the suite, so a run starts exactly the jobs it +needs and shows no skipped job for a path it did not take. ### Daily -Daily is the recurring build and artifact-publication path. +Daily builds every entry in the matrix, and everything in the staging archive +comes from a run of it. -- The scheduled run resolves the full `Daily` matrix. -- A manual run selects one **Build scope**: - - **Full matrix** builds every configured variant and suite. - - **Selected variant (all suites)** builds every configured suite for one variant. - - **Selected variant and suite** builds one isolated matrix leg. +- The scheduled run resolves the full matrix. +- A manual run says which entries to build in one **Builds** field: + - `all`, the default, builds every configured variant and suite. + - A comma-separated list of build names builds those entries, e.g. + `qcom-next-trixie,qcom-next-debug-forky`. A name matching no entry fails + the run rather than narrowing it, and both families are selected from one + list, so a mixed list starts Debian and Ubuntu legs from one dispatch. - `latest_tag` resolves the newest matching dated tag; `branch_tip` resolves the configured branch directly. - Debian suites build in Debusine, then their `.deb` outputs are downloaded and uploaded to the configured S3 bucket. +- Debian entries are then promoted into the staging workspace with Debusine's + `package-publish` workflow, making them installable from that archive. +- Debian builds resolve their Build-Depends against `qli` alone. The nightly + is the build whose output is published, so anything it builds against is + something the published kernel depends on; reading `qli-staging` here would + let a kernel be published depending on a `-dkms` package that has not been + released. A PR build resolves against `qli qli-staging`, because nothing it + produces is promoted and a kernel and the module it needs should be + reviewable together. - `resolute` stays on the Docker-based Ubuntu path and uploads its package outputs to the existing temporary-package S3 location. +Because the package version is a function of the resolved ref and the entry's +revision, a night on which the tracked tag has not moved would rebuild a +version the archive already has and then fail promoting the duplicate. The +`prepare` job therefore asks the target workspace whether it already holds the +version this run would produce, and skips the build, the S3 publication and the +promotion when it does. A run that skips this way is green: nothing was wrong, +there was simply nothing new upstream. + +Where a build is published belongs to the run, not to the entry: the nightly +names the staging workspace, and `pr-build.yml` names nothing, so a pull +request's kernel is built and tested but reaches no archive. Everything a +`daily` dispatch can say about a build comes from the matrix entry, so there is +no way to dispatch a build that differs from the nightly one at all. + ### Release -Release is the controlled promotion path. - -- It is manual only and uses one **Release scope** for a kernel variant: - - **Selected variant (all suites)** is the normal release action and promotes - every configured Release suite for that variant. - - **Selected variant and suite** promotes one configured Release suite for - that variant when a targeted action is required. -- It uses the pinned `branch_or_tag` from the selected `Release` matrix row; it - never resolves a newest tag. -- Debian source and binary artifacts are built in per-variant, per-suite - Debusine CI workspaces. -- Successful builds are promoted with Debusine's `package-publish` workflow to - the `qli` workspace, where they are available through the production Debusine - APT repository. -- The Release job runs in the **Production** GitHub environment. This provides - the release credential and enforces the required approval gate before - promotion to `qli`. - -Direct `build-kernel-deb.yml` dispatches are build-only. Release promotion is -initiated exclusively by `release.yml`, which owns the target workspace and -production release controls. +A release builds the ref it releases, and publishes the result into `qli`. + +- Every value describing a release lives in `ci/build-matrix.yaml` under + `releases`: the pinned ref, the suites, the packaging, and the + `target_workspace` each entry publishes into. A dispatch only says which + entries to run, in the same **Builds** field Daily uses. +- Updating a release is a pull request that changes `branch_or_tag`. That is + what puts the ref that ships under review, rather than a version typed into + a dispatch form at the moment of releasing. +- `ref_strategy` must be `pinned_ref`. `latest_tag` would make two dispatches + of one entry release different kernels, and `branch_tip` would make them + release whatever the branch had reached. +- Debian suites only, and `resolve-matrix.py` rejects anything else: promotion + runs through Debusine, and the Ubuntu path has no workspace to promote into. +- The `promote` job runs in the **Production** environment, so whatever + approval that environment requires stands between the run and the archive. +- Build-Depends resolve against `qli` alone, as the nightly's do, so a released + kernel cannot depend on a `-dkms` package that has not itself been released. + +It rebuilds rather than promoting a nightly because there is nothing left to +promote from: the CI workspace a nightly ran in is named after that run and does +not outlive it. What reaches `qli` is therefore the artifact this run built and +tested, from a ref that cannot have moved since it was reviewed. + +### Publishing + +Promotion happens inside the build, in the `promote` job of +`build-kernel-debian.yml`, using Debusine's `package-publish` workflow. + +- It reads the ephemeral CI child workspace the build ran in and publishes the + source and binary artifacts into the workspace the caller named. +- It is in-run by necessity. `lib/build` names that workspace + `-gh----`, creating it fresh for the + run and not keeping it afterwards, so a later workflow would have neither the + name nor the contents to promote. Publishing is part of the build or it does + not happen. +- Only the Debian family reaches it, because promotion runs through Debusine + and the Ubuntu path does not build there. +- Which GitHub environment it runs in is the caller's, through + `promote-environment`. Daily leaves it at **Staging** and promotes + unattended, because `qli-staging` is where an unreviewed nightly kernel + belongs; Release passes **Production**, so an approval stands in front of + `qli`. + +A nightly promotion changes no file in this repository: there is no ref to pin +and no entry to bless, because what is published is what was built. A release +does, and that is the difference between them — the ref it ships is written +down and reviewed before the run that ships it. + +#### Moving the nightly to `qli` + +`qli-staging` is the nightly's destination, set by `DEBUSINE_STAGING_WORKSPACE` +and defaulted in [daily.yml](.github/workflows/daily.yml). Pointing that +variable at `qli` would publish every night's kernel straight into the released +archive with nothing in between, and the approval gate that +`promote-environment` provides would then have to be applied to every nightly +run — stopping each of them to wait for one. The two archives, with +`release.yml` between them, are what keep nightlies unattended and `qli` gated. ## Matrix Model -`ci/build-matrix.json` is an object with two top-level keys: `deliveries` -(the matrix rows) and `suite_suffix_mapping` (matrix-wide Debian suffix -policy, shared by every variant and delivery type). `ci/scripts/resolve-matrix.sh` -validates the document, requires each `kernel_variant` to have exactly one -`Daily` and one `Release` row in `deliveries`, filters by delivery type, and -flattens each `suites` array into independent suite legs. Each leg carries -its own values for: +`ci/build-matrix.yaml` is a mapping with two top-level keys, `builds` and +`releases`. One entry in either is one generated package, so there is no +expansion step: `ci/scripts/resolve-matrix.py` validates the whole document, +selects the entries matching the requested names, variant, and suite, and hands +them to the workflow matrix as they stand. + +The two lists have the same shape and are validated separately, so a name may +appear in both and means the same build in each. `builds` is what Daily runs +nightly; `releases` is what `release.yml` can ship, selected with +`--releases`. Both are validated on every invocation whichever is asked for, so +a broken release entry fails a nightly run rather than waiting to be found by +whoever next tries to release. Each entry carries: | Field | Purpose | | --- | --- | -| `kernel_variant` | Stable identifier for a separately packaged kernel variant. Lowercase letters, digits, and internal hyphens only. | -| `type` | `Daily` or `Release`. | -| `suites` | Suites to flatten into individual build legs. Each must have a `suite_suffix_mapping` entry. | +| `name` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique across the matrix. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only, and not `all`, which a dispatch reads as every entry. | +| `suite` | The one suite this entry builds for. | +| `flavour` | The kernel's own identity, and the only matrix field that reaches the built kernel. `derive-localversion.sh` makes it the LOCALVERSION suffix, so two flavours built from one ref get distinct kernel releases and their `linux-image` packages coexist. All entries sharing a flavour build the same package for different suites. | | `git_clone` | Kernel source repository. | | `branch_or_tag` | Source branch or pinned tag, according to `ref_strategy`. | | `ref_strategy` | `latest_tag`, `branch_tip`, or `pinned_ref`. | | `tag_pattern` | Required only for `latest_tag`; matching tags must end in `-YYYYMMDD`, which determines newest-first ordering. | | `srcpkg` | Debian source package name. | | `binpkg` | Kernel image metapackage name. | -| `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per array element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.sh` joins it into the comma-separated `kernel-config` workflow input. | -| `dkms` | Out-of-tree DKMS modules built against this kernel and bundled into its `linux-image` package, one per array element, each named as the stem of its `-dkms` package (e.g. `kgsl`). Empty bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.sh` joins it into the comma-separated `dkms` workflow input. | -| `debian_version_stub` | Base Debian revision, shared by a variant's Daily and Release rows. Must not end in `~`; the suite suffix is derived, not stored here. | -| `debian_version_suffix` | `~` for Daily rows, empty for Release rows. Documents the delivery-type half of the revision formula on the row itself; `resolve-matrix.sh` rejects a row where this disagrees with `type`, but derivation always computes this suffix from `type`, never reads this field. | +| `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per list element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.py` joins it into the comma-separated `kernel-config` workflow input. | +| `dkms` | Out-of-tree DKMS modules built and bundled into the image package, one per list element, each named without the `-dkms` suffix (e.g. `kgsl`). Each needs a `-dkms` package in the suite being built for, so this varies between suites. An empty list bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.py` joins it into the comma-separated `dkms` workflow input, which reaches `prepare-source.sh --dkms`; see [debian/README.md](debian/README.md) for what the packaging does with it. | +| `debian_revision` | The Debian revision this package is built at, stated outright. Carried into the archive as built, because publishing promotes the artifact rather than rebuilding it. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | -| `target_workspace` | Debusine destination for Release entries only. | - -`dkms` currently has one exception the matrix cannot express: `build-kernel-deb.yml` -replaces the resolved list with `kgsl` on Ubuntu-family legs, so `camx` and -`iris-vpu` are bundled on Debian suites only. That override is temporary and goes -away once the matrix gains per-suite `dkms` lists. - -`target_workspace` is required for `Release` and rejected for `Daily`. -`tag_pattern` is required for `latest_tag` and rejected for other strategies. -The resolver selects the most recent trailing `YYYYMMDD` date, and rejects -duplicate suites and malformed variant identifiers before any build jobs -start. It also rejects a matrix where any configured suite has no -`suite_suffix_mapping` entry, where two suites share the same suffix, where a -suffix is non-empty and doesn't start with `~`, where a variant's Daily -and Release rows disagree on `debian_version_stub`, or where a row's -`debian_version_suffix` doesn't match what its `type` implies — all before -any build job starts. - -Each flattened leg's final `debian_revision` is derived by -`ci/scripts/derive-debian-revision.sh` from `debian_version_stub`, -`suite_suffix_mapping[suite]`, and the delivery type -(`stub + suffix + "~"` for Daily, `stub + suffix` for Release). This script is -the single implementation of the formula: `resolve-matrix.sh` calls it once -per flattened leg, and `build-kernel-deb.yml`'s direct-dispatch path (which -has no full-matrix context) calls the same script for the one suite it was -given. - -Each leg has a distinct prepared-source artifact, Debusine child workspace, and -S3 path keyed by `kernel_variant + suite`. This prevents two variants that both +| `target_workspace` | **`releases` only, and required there.** The Debusine workspace this entry publishes into. It is the one field a `builds` entry may not carry: where a nightly goes follows from why it is running, and is the calling workflow's to decide, while a release exists precisely to put one ref into one archive. | + +`resolve-matrix.py` rejects the matrix — before any build job starts — where an +entry has an unknown field or a missing required one, a malformed variant or +suite identifier, an unknown `ref_strategy`, a `tag_pattern` without +`latest_tag`, a `kernel_config` fragment that escapes the kernel source root +or collides with another fragment's filename, a `dkms` entry that is not a package name stem or repeats, +or a `debian_revision` that is not a valid Debian revision. + +A `releases` entry is held to two rules of its own: `ref_strategy` must be +`pinned_ref`, so releasing twice releases the same kernel, and `suite` must be +a Debian one, because only that path can promote into a workspace. + +Because entries are written out in full, the resolver also checks the +invariants that span them, which is what makes the duplication safe to read at +face value: + +- No two entries share a `name` — a name identifies exactly one build, so a run + cannot produce two jobs with one name and a dispatch cannot be ambiguous. +- A flavour's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those + decide what the package *is*, and the entries differ only in where it goes. +- A flavour's entries for one suite agree on `dkms`. The module set depends on + which `-dkms` packages the target archive carries, so it varies between + suites but not within one. +- A flavour's entries agree on `git_clone`, `branch_or_tag`, `ref_strategy` and + `tag_pattern`, so a forgotten suite cannot quietly ship a different kernel + from its siblings. +- No `srcpkg` or `binpkg` is shared between flavours, and no two entries build + the same `srcpkg` at the same `debian_revision`. + +The build workflows pass their entry's own `debian_revision` through, and the +`prepare-kernel-source` action requires it: there is no caller without an +entry in hand, so there is no fallback to look one up. + +Each entry has a distinct source-package artifact, Debusine child workspace, +and S3 path keyed by `flavour + suite`. This prevents two flavours that both build, for example, `trixie` from consuming or publishing each other's inputs -or outputs. +or outputs. They key on `flavour`, not on the build's `name`, so renaming a +build never moves a published artifact. -Daily S3 outputs use these layouts, where `` is +S3 outputs use these layouts, where `` is `-`: ```text -/pkg/debusine///// -/pkg/temp///// +/pkg/debusine///// +/pkg/temp///// ``` +Each holds the `.deb` files, and a `source/` directory beneath it holds the +source package they were built from: the `.changes`, `.dsc`, `.debian.tar.xz` +and `.orig.tar.gz`. + The first layout is for Debian/Debusine builds; the second is for Ubuntu Docker -builds. Consumers must select the intended kernel variant and suite. +builds. Consumers must select the intended flavour and suite. Supporting scripts keep workflow YAML small and testable: | Script | Responsibility | | --- | --- | -| `ci/scripts/resolve-matrix.sh` | Validates and flattens matrix rows. | +| `ci/scripts/resolve-matrix.py` | Validates the delivery matrix and selects the entries to build. Needs PyYAML (`python3-yaml`). | | `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | -| `ci/scripts/derive-localversion.sh` | Derives `LOCALVERSION` from the variant and resolved kernel ref. | -| `ci/scripts/derive-debian-revision.sh` | Derives the final suite-specific `debian_revision` from `debian_version_stub`, `suite_suffix_mapping`, and delivery type. | +| `ci/scripts/derive-localversion.sh` | Derives the version fields from the flavour, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version: the tag's date, or the HEAD commit date for a branch-tip build. Scheme and rationale: [docs/version.md](docs/version.md). | ## Architecture @@ -252,17 +339,24 @@ This document covers the CI generator. For the packaging internals: `debian/rule targets, the config fragment merge pipeline, DKMS module bundling and the produced package layout see [debian/README.md](debian/README.md). +Both branches below are taken in the caller. The family follows from the +entry's suite; the promotion follows from which workflow is running. By the +time a build workflow starts, there is nothing left to decide. + ```mermaid flowchart LR - IN["Matrix variant + suite input"] --> R{"Resolve suite family"} + IN["Matrix entries"] --> R{"resolve-matrix.py\n--family"} - R -->|"trixie · forky"| DEB["Debian path\nbuild-kernel-debusine.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] - R -->|"resolute"| UBU["Ubuntu path\nbuild-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] + R -->|"debian: trixie · forky"| DEB["build-kernel-debian.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] + R -->|"ubuntu: resolute"| UBU["build-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] - DEB --> DOUT{"Build type"} - DOUT -->|Daily| S3["Download .deb files\nPublish to S3"] - DOUT -->|Release| QLI["Promote source and binaries\nto qli"] + DEB --> S3["Download .deb files\nPublish to S3"] + DEB --> TW{"Which caller"} + TW -->|"daily.yml"| STG["Promote to qli-staging\nStaging environment"] + TW -->|"release.yml"| REL["Promote to qli\nProduction environment"] + TW -->|"pr-build.yml"| NONE["No archive"] UBU --> US3["Publish .deb files to S3"] + ``` ## For CI Maintainers @@ -273,90 +367,94 @@ flowchart LR flowchart TD subgraph triggers[Triggers] A1["daily.yml\nScheduled full matrix"] - A2["daily.yml\nManual full or filtered variant + suite"] - A3["release.yml\nManual full or filtered variant + suite"] - A4["build-kernel-deb.yml\nManual one-off build"] + A2["daily.yml\nManual: all or named builds"] + A3["release.yml\nManual: pinned refs from releases"] + A5["pr-build.yml\nFull matrix on every PR"] end subgraph matrix[Matrix entry points] - B1["Daily configure-matrix\nFlatten Daily rows"] - B2["Daily variant + suite legs\nqcom-next / trixie · forky · resolute\nqcom-next-debug / trixie · forky"] - B3["Release configure-matrix\nFlatten Release rows"] - B4["Release variant + suite legs\nqcom-next / trixie · forky\nqcom-next-debug / trixie · forky"] + B1["configure-matrix\nEntries, split by family"] + B2["build-debian legs\nqcom-next · qcom-next-debug · qcom-arduino\nmainline · next / trixie · forky"] + B5["build-ubuntu legs\nqcom-next / resolute"] end - subgraph orchestrator[build-kernel-deb.yml] - C1["resolve\nClassify suite family"] - C2["prepare\nClone selected kernel ref\nRun prepare-source.sh\nUpload kernel-srcpkg-variant-suite"] - C3["debusine-build\nDebian suites only"] - C4["ubuntu-build\nUbuntu suites only"] + subgraph build[One build workflow per leg] + C2["prepare\nprepare-kernel-source action\nClone ref, build the source package\nSkip the run if the version is published"] + C3["build\ndebusine-build action"] + C4["build\nprepare-kernel-source action, then\nbuild-kernel.sh --dsc in Docker"] + C5["publish\nDownload .deb files, upload to S3"] + C6["promote\nlib/release into the caller's workspace"] end subgraph outputs[Outputs] - D1["Daily S3 artifacts"] - D2["Release qli APT repository"] + D1["S3 artifacts"] + D3["qli-staging APT repository"] + D4["qli APT repository"] end A1 --> B1 A2 --> B1 - A3 --> B3 - B1 --> B2 --> C1 - B3 --> B4 --> C1 - A4 --> C1 - C1 --> C2 - C2 --> C3 & C4 - C3 --> D1 & D2 + A3 --> B1 + A5 --> B1 + B1 --> B2 & B5 + B2 --> C2 + B5 --> C4 + C2 --> C3 + C3 --> C5 & C6 C4 --> D1 + C5 --> D1 + C6 --> D3 & D4 ``` +A leg runs every job drawn under it except `promote`, which only a caller +naming a workspace reaches — so a PR build stops at S3. Which archive `promote` +reaches, and whether it waits for an approval first, is that caller's too: the +nightly goes to `qli-staging` unattended, and a release to `qli` through the +Production environment. + ### Prepare stage ```mermaid flowchart LR - K["Matrix-selected kernel repository\nDaily: latest tag or branch tip\nRelease: pinned ref"] --> PS + K["Matrix-selected kernel repository\nlatest tag, branch tip, or pinned ref"] --> PS M["pkg-linux-qcom\ndebian/ and ci/ from this commit"] --> PS - PS["prepare-source.sh\n\nInject debian/\nApply all config-available fragments plus any extras\nGenerate control, changelog, localversion, pkgversion"] --> TAR - TAR["tar czf kernel-srcpkg-variant-suite.tar.gz\nPreserves execute permissions"] --> ART - ART["GitHub Actions artifact\nOne prepared source tree per variant + suite"] + PS["prepare-source.sh\n\nInject debian/\nApply all config-available fragments plus any extras\nGenerate control, changelog, localversion, pkgversion"] --> BSP + BSP["build-source-package.sh\n\ngit archive the commit → .orig.tar.gz\ndpkg-source -b → .dsc, .debian.tar.xz\ndpkg-genchanges -S → .changes"] --> ART + ART["GitHub Actions artifact\nsource-package-variant-suite\nOne .changes set per variant + suite"] ``` -> **Why `tar.gz`?** `actions/upload-artifact` uses zip internally, which strips -> Unix execute bits. Kernel build scripts require those permissions. The tar -> archive preserves them between the prepare and build jobs. +Both steps run in the pkg-builder container on the self-hosted runner. The +orig tarball is a function of the kernel commit: `git archive` gives every +entry the commit's timestamp and root ownership, and `gzip -n` writes no +timestamp, so two runs on one commit write one tarball and two suites that +differ only in Debian revision share it. `build-source-package.sh` checks +that the commit it archives is the one the version names (`~g`) and +that the tree matches that commit outside `debian/`. See +[debian/README.md](debian/README.md#building-a-source-package) for the +local equivalent. -### Debian Daily path +### Debian build path ```mermaid flowchart LR - ART["kernel-srcpkg-variant-suite\nartifact"] --> GSP + ART["source-package-variant-suite\nartifact"] --> SUBMIT subgraph source[GitHub build job: debusine-pkg-builder container] - GSP["generate-source-package\nDEBUSINE_ASSEMBLE_ORIG=true\n\nCreate .orig.tar.gz\nRun dpkg-buildpackage -S\nProduce .dsc"] --> SUBMIT - SUBMIT["lib/build\nCreate CI child workspace\nSubmit source package to Debusine"] + SUBMIT["lib/build\nCreate CI child workspace\nSubmit the .dsc to Debusine"] end SUBMIT --> DEB["Debusine\nBuild binary packages"] DEB --> WS["Unique variant + suite workspace"] - subgraph publish[Daily publish job] + subgraph publish[publish job] WS --> APT["generate-apt-config\nchdist isolated APT environment\nDownload .deb files"] - APT --> S3["S3\nDaily package artifacts"] + APT --> S3["S3\npackage artifacts"] end -``` - -### Debian Release path - -```mermaid -flowchart LR - ART["kernel-srcpkg-variant-suite\nartifact"] --> GSP["generate-source-package\nProduce .dsc"] - GSP --> SUBMIT["lib/build\nSubmit source package to a unique\nDebusine CI child workspace"] - SUBMIT --> DEB["Debusine\nBuild binary packages"] - DEB --> WS["CI workspace\nsource and binary artifacts"] - subgraph release[Release job: Production GitHub environment] + subgraph promote[promote job: only when the caller named a workspace] WS --> PROMOTE["lib/release\nStart package-publish"] - PROMOTE --> QLI["qli\nProduction Debusine APT repository"] + PROMOTE --> STG["qli-staging (daily)\nqli (release)\nDebusine APT repository"] end ``` @@ -364,19 +462,17 @@ flowchart LR ```mermaid flowchart LR - ART["kernel-srcpkg-variant-suite\nartifact"] --> EXT + ART["source-package-variant-suite\nartifact"] --> BK subgraph build[Ubuntu build job] - EXT["Extract prepared source tree\n--strip-components=1"] --> BK - BK["build-kernel.sh\n--skip-prepare\n--local-source\n--build-mode docker\ndpkg-buildpackage -b"] --> S3 + BK["build-kernel.sh --dsc\n--build-mode docker\nsbuild in the suite's pkg-builder image"] --> S3 end - S3["S3\nDaily package artifacts"] + S3["S3\npackage artifacts"] ``` -`--skip-prepare` is safe because `prepare-source.sh` has already generated the -packaging metadata and applied the config fragments before the artifact is -created. +Nothing is cloned or prepared here: the `.dsc` the prepare job reported is +what sbuild builds, as it is what Debusine builds on the Debian path. ## Packages @@ -399,6 +495,17 @@ For the current matrix, package generation produces: names. Only the Debian version field converts it to `~rcN`, so a release candidate correctly sorts before the corresponding final kernel release. +Every build names both its snapshot and the commit it was cut from: + +| | Format | Example | +| --- | --- | --- | +| Kernel release (`uname -r`) | `+-[.]-g` | `7.2.0-rc7+qcom-next-20260826.1-g011a82096bee` | +| Debian version | `+git[.]~g-` | `7.2.0~rc7+git20260826.1~g011a82096bee-0qli1~bpo13+1` | + +The two strings spell the same fields differently because they are compared by +different rules — `+` and `~` are both load-bearing, not stylistic. See +[docs/version.md](docs/version.md) before changing either. + `KVER_EXTRA` is supported for explicit suffixes such as `-ci42` or `-local`. The packaging rules verify that the declared versioned image package matches the resolved kernel release and fail instead of creating inconsistent metadata. @@ -416,37 +523,42 @@ out-of-tree module builds are required. ## Manual Builds -Use **Actions** → **build-kernel-deb** for a one-off build. It is an explicit -override workflow, not a matrix-derived delivery flow: use `daily.yml` and -`release.yml` for normal Daily and Release operations. - -`kernel-variant`, `suite`, and `ref-strategy` are the required build selection. -All remaining package, configuration, and PR inputs are advanced overrides for -validation or debugging. Variant and suite are free-text matrix values rather -than static dropdowns, so adding a matrix entry never requires editing the -workflow UI. - -The available inputs are: +Use **Actions** → **daily** → **Run workflow** for a one-off build. It names +what to build in one **Builds** field, and takes nothing else. | Input | Default | Purpose | | --- | --- | --- | -| `kernel-variant` | `qcom-next` | Stable variant identifier used in artifact and workspace identity. | -| `suite` | `trixie` | Target suite. | -| `ref-strategy` | `latest_tag` | `latest_tag`, `branch_tip`, or `pinned_ref`. | -| `kernel-branch` | `qcom-next` | Branch for `branch_tip`, or immutable ref for `pinned_ref`; ignored by `latest_tag`. | -| `tag-pattern` | `qcom-next-*` | Tag glob for `latest_tag`; ignored by `branch_tip` and `pinned_ref`. | -| `kernel-url` | `qualcomm-linux/kernel` | Advanced alternate kernel repository. | -| `srcpkg` | `linux-qcom-next` | Advanced source package identity override. | -| `binpkg` | `linux-image-qcom-next` | Advanced image metapackage identity override. | -| `kernel-config` | Empty | Advanced extra fragments applied on top of all of `debian/config-available/`, e.g. `intree:arch/arm64/configs/qcom_debug.config`. | -| `debian-version-stub` | `0qli` | Advanced Debian version stub. The selected suite's mapped suffix and a Daily-style trailing `~` are applied automatically; direct builds always use Daily semantics since they are build-only and non-promoting. | -| `localversion` | Auto-derived | Advanced explicit `LOCALVERSION` override. | -| `kver-extra` | Empty | Advanced kernel-release suffix. | -| `debug-build` | `false` | Advanced debug configuration toggle. | - -The workflow also supports advanced Qualcomm-only PR overrides for validation -builds. Direct builds are artifact builds; Release promotion is performed only -through `release.yml`. +| `builds` | `all` | `all` selects every entry. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | + +Everything else about a build — its suite, flavour, kernel repository and ref, +package names, config fragments, DKMS modules and Debian revision — comes from +the entry, so there is nothing to retype and nothing to get wrong. Build names +are free-text matrix values rather than a static dropdown, so adding a matrix +entry never requires editing the workflow UI, and a name that matches no entry +fails the run with the list of names that do. `daily` routes each selected +entry to the workflow that builds its family, so one dispatch can name Debian +and Ubuntu builds together. + +A `daily` dispatch is not a lesser run: it publishes to S3 and promotes its +Debian entries into the archive exactly as the scheduled run does, because a +dispatch cannot describe a build that differs from the nightly one. The build +workflows themselves (`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`) are +`workflow_call` only and cannot be dispatched: one run of each is one matrix +entry, and a reusable workflow cannot fan itself out over a list. + +### Releasing + +Use **Actions** → **release** → **Run workflow**. It takes the same **Builds** +field, selecting from the matrix's `releases` list rather than from `builds`, +and nothing else — the ref, the packaging and the destination archive are all +the entry's. + +Release a kernel by opening a pull request that sets `branch_or_tag` on the +entries being released, merging it, and dispatching `release` for those names. +The run stops at the `promote` job for the Production environment's approval, +having already built, published to S3 and printed what it is about to release +in the run summary, so the approval is given against the refs and versions in +front of you. ## Configuration @@ -454,57 +566,75 @@ through `release.yml`. | Variable | Purpose | | --- | --- | -| `ARTIFACT_S3_BUCKET` | S3 bucket for Daily Debian artifacts and Ubuntu build artifacts. | +| `ARTIFACT_S3_BUCKET` | S3 bucket for Debian and Ubuntu build artifacts. | | `DEBUSINE_HOST` | Production Debusine host. | | `DEBUSINE_SCOPE` | Debusine scope. | | `DEBUSINE_PARENT_WORKSPACE` | Parent workspace used to create per-run CI child workspaces. | +| `DEBUSINE_STAGING_WORKSPACE` | Workspace the nightly build promotes into. Defaults to `qli-staging`; see [Moving the nightly to `qli`](#moving-the-nightly-to-qli) before changing it. A release names its workspace in the matrix instead, so this does not affect it. | ### Secrets | Secret | Scope | Purpose | | --- | --- | --- | | `DEBUSINE_USER` | Repository | User for Debusine archive and signing-key access. | -| `DEBUSINE_TOKEN` | Repository | Token for Debusine build and artifact operations. | -| `DEBUSINE_RELEASE_TOKEN` | Production environment | Token used only to promote Release artifacts to `qli`. | +| `DEBUSINE_TOKEN` | Repository | Token for Debusine build and artifact operations, including the nightly promotion into `qli-staging`. | -The Debian build and Release jobs select the **Production** GitHub environment. -This makes environment-scoped release credentials available to the promotion job -and keeps production approval controls in the workflow path. +The `build` job of `build-kernel-debian.yml` always selects the **Staging** +GitHub environment. The `promote` job selects whichever its caller names +through `promote-environment`: **Staging** for the nightly, which is what lets +it promote unattended, and **Production** for a release, which is what makes it +wait for an approval. + +`DEBUSINE_TOKEN` is what both promotions authenticate with, so it needs write +access to `DEBUSINE_STAGING_WORKSPACE` and to every `target_workspace` the +`releases` list names — `qli` included. `DEBUSINE_RELEASE_TOKEN` is not read by +any workflow; a release is gated by the Production environment rather than by a +credential of its own. ## Maintaining the Matrix To add a kernel variant: -1. Add exactly two rows to `deliveries` with the same `kernel_variant`: one - `Daily` and one `Release`. -2. Define all package identity, source/ref strategy, configuration, - `debian_version_stub`, and suite values in both rows. Do not rely on - another variant's values. `srcpkg`, `binpkg`, and `debian_version_stub` - must remain identical across the pair. Set `debian_version_suffix` to `~` - on the Daily row and `""` on the Release row; `resolve-matrix.sh` rejects - the pair if either disagrees with its row's `type`. -3. Use `latest_tag` with a dated tag glob or `branch_tip` for Daily. Use - `pinned_ref` for Release, and update that ref through a reviewed PR. -4. Give the variant distinct `srcpkg` and `binpkg` values. Set the Release - `target_workspace` explicitly. -5. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use +1. Add one entry to `builds` per package the variant should produce: one per + suite, each spelling out all of its own fields, and each with a `name` + unique across the matrix. Do not rely on another variant's values. +2. Give them all the same `flavour`, distinct from every other flavour's — it + becomes the LOCALVERSION suffix, so this is what lets the new kernel install + alongside the existing ones. Keep `srcpkg`, `binpkg`, `kernel_config` and + the ref fields identical across every entry for the flavour. + `resolve-matrix.py` rejects the matrix if they drift apart. +3. Use `latest_tag` with a dated tag glob, or `branch_tip`, to track a moving + upstream; `pinned_ref` freezes the variant on one ref. +4. Give the variant distinct `srcpkg` and `binpkg` values. +5. Give each entry a `debian_revision` that sorts where its suite belongs + relative to the others and collides with no other entry building the same + `srcpkg`. +6. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use the Docker path. -6. Run a filtered Daily validation for the new variant, then its full Daily and - Release flows. +7. Run a filtered daily validation for the new variant, then a full daily run. + Nothing further is needed to publish it: a successful nightly build of a + Debian entry promotes itself. To add a new suite (for an existing or new variant): -1. Add an entry for it to the shared top-level `suite_suffix_mapping`, empty - or starting with `~`, and distinct from every other suite's suffix. -2. Add the suite to the `suites` array of the relevant Daily and/or Release - rows. `resolve-matrix.sh` rejects any configured suite with no mapping - entry before any build job starts. -3. Choose the suffix so the suite sorts where it belongs relative to the - others for the same delivery type (see the ordering discussion in - [Overview](#overview)). - -No workflow dispatch choices need to be updated: manual Daily and Release -inputs accept matrix-defined variant and suite strings. +1. Add one entry for the suite, copying one of the variant's existing entries + and changing `suite` and `debian_revision`. +2. Choose the revision so the suite sorts where it belongs relative to the + others (see the ordering discussion in [Overview](#overview)), and so it + does not collide with another entry building the same `srcpkg`. + +To make a variant releasable, add the matching entries to `releases`: the same +fields, plus `target_workspace`, with `ref_strategy: pinned_ref` and +`branch_or_tag` naming the ref that ships. A variant with no `releases` entries +is built nightly and never shipped, which is the right state for a topic branch +or a tracking build. + +No workflow dispatch choices need to be updated: the daily and release +dispatches take build names as free text, so a new entry is dispatchable by +name, and is picked up by `all`, as soon as it is merged. + +Run `ci/scripts/resolve-matrix.py` locally to validate a matrix change before +pushing it; it validates the whole document regardless of what it selects. ## Contributing diff --git a/build-kernel.sh b/build-kernel.sh index 23375748..a0dd5668 100755 --- a/build-kernel.sh +++ b/build-kernel.sh @@ -8,9 +8,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DEFAULT_REPO="git@github.com:qualcomm-linux/kernel.git" +DEFAULT_REPO="https://github.com/qualcomm-linux/kernel" DEFAULT_BRANCH="qcom-next" DEFAULT_DISTRO="trixie" +DEFAULT_FLAVOUR="qcom-next" DEFAULT_BUILD_MODE="docker" KERNEL_DIR="$SCRIPT_DIR/kernel-source" OUTPUT_BASE_DIR="$SCRIPT_DIR/kernel-build" @@ -36,13 +37,28 @@ OPTIONS: -l, --latest-tag Select the latest qcom-next-* tag automatically -b, --branch BRANCH Branch to use (default: $DEFAULT_BRANCH) -r, --repo URL Kernel repository URL (default: $DEFAULT_REPO) + --dsc FILE Build from an existing Debian source package + instead of a kernel tree: no clone, no prepare. + The .orig.tar.gz and .debian.tar.xz the .dsc + names must sit beside it, as build-source-package.sh + and CI leave them. + + What to build: + --source-package Stop after building the Debian source package + (.orig.tar.gz, .dsc, .changes) into the output + directory, with build-source-package.sh. The orig + tarball is reproducible: see that script. Build + binaries from it later with --dsc. Version control: - --localversion TAG LOCALVERSION suffix (e.g. qcom-next-20260312) - Auto-detected from git tag when using --local-source + --flavour NAME Kernel flavour carried in LOCALVERSION + (default: $DEFAULT_FLAVOUR). Ignored with --localversion. + --localversion SUFFIX LOCALVERSION suffix (e.g. +qcom-next-20260312-g07f50dc44edd) + Derived from the checked-out tag or branch if not given, + by prepare-source.sh, the same way CI derives it. --kver-extra SUFFIX Extra suffix appended to the derived KVER, e.g.: --kver-extra -mybuild - Results in: 7.0.0-rc2-qcom-next-20260312-mybuild + Results in: 7.0.0-rc2+qcom-next-20260312-g07f50dc44edd-mybuild Useful for CI build IDs or local user builds. Build control: @@ -82,6 +98,8 @@ EXAMPLES: $0 --local-source /path/to/kernel --kver-extra -mybuild $0 --latest-tag --kernel-config docker,systemd-boot $0 --latest-tag --dkms kgsl + $0 --latest-tag --source-package + $0 --dsc kernel-build/trixie/linux-qcom-next_*.dsc DISTRIBUTIONS: noble Ubuntu 24.04 LTS @@ -95,10 +113,10 @@ EOF # Defaults TAG=""; LATEST_TAG=false; BRANCH="$DEFAULT_BRANCH"; REPO="$DEFAULT_REPO" -DISTRO="$DEFAULT_DISTRO"; BUILD_MODE="$DEFAULT_BUILD_MODE" +DISTRO="$DEFAULT_DISTRO"; BUILD_MODE="$DEFAULT_BUILD_MODE"; FLAVOUR="$DEFAULT_FLAVOUR" LOCALVERSION=""; KVER_EXTRA=""; PROFILES=""; CLEAN=false LOCAL_SOURCE=""; ENABLE_CONFIGS="squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can"; SKIP_PREPARE=false -DKMS_MODULES="" +DKMS_MODULES=""; DSC=""; SOURCE_PACKAGE=false while [[ $# -gt 0 ]]; do case $1 in @@ -111,7 +129,10 @@ while [[ $# -gt 0 ]]; do --debian-dir) DEBIAN_DIR="$2"; shift 2 ;; -d|--distro) DISTRO="$2"; shift 2 ;; --local-source) LOCAL_SOURCE="$2"; shift 2 ;; + --dsc) DSC="$2"; shift 2 ;; + --source-package) SOURCE_PACKAGE=true; shift ;; --docker-build) DOCKER_PKG_BUILD="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; --profiles) PROFILES="$2"; shift 2 ;; @@ -142,20 +163,35 @@ VALID_MODES=(docker native sbuild) exit 1 } -# Locate docker_deb_build.py (docker mode) +# --dsc replaces the tree: everything that selects or prepares one is +# meaningless beside it, and so is asking for the source package it already is. +if [[ -n "$DSC" ]]; then + [[ -z "$LOCAL_SOURCE$TAG" && "$LATEST_TAG" == false && "$SKIP_PREPARE" == false && "$SOURCE_PACKAGE" == false ]] || { + log_error "--dsc cannot be combined with --local-source, --tag, --latest-tag, --skip-prepare or --source-package" + exit 1 + } + [[ -f "$DSC" ]] || { log_error "Source package not found: $DSC"; exit 1; } + DSC="$(cd "$(dirname "$DSC")" && pwd)/$(basename "$DSC")" +fi + +# Locate docker_deb_build.py (docker mode). Building binaries from a tree goes +# through it, so it must exist for that. Building from a .dsc runs sbuild in +# the image directly and only needs it to build the image when the image is +# missing, and --source-package builds no binaries at all, so for those it is +# looked for and not required. if [[ "$BUILD_MODE" == "docker" && -z "$DOCKER_PKG_BUILD" ]]; then for p in "$HOME/docker-pkg-build/docker_deb_build.py" \ "$SCRIPT_DIR/docker-pkg-build/docker_deb_build.py" \ "$(which docker_deb_build.py 2>/dev/null || true)"; do [[ -x "$p" ]] && { DOCKER_PKG_BUILD="$p"; break; } done - [[ -z "$DOCKER_PKG_BUILD" ]] && { + [[ -z "$DOCKER_PKG_BUILD" && -z "$DSC" && "$SOURCE_PACKAGE" == false ]] && { log_error "docker_deb_build.py not found. Use --docker-build, set DOCKER_PKG_BUILD, or use --build-mode native." exit 1 } fi [[ "$BUILD_MODE" == "docker" && -d "$DOCKER_PKG_BUILD" ]] && DOCKER_PKG_BUILD="$DOCKER_PKG_BUILD/docker_deb_build.py" -[[ "$BUILD_MODE" == "docker" && ! -x "$DOCKER_PKG_BUILD" ]] && { log_error "Not executable: $DOCKER_PKG_BUILD"; exit 1; } +[[ "$BUILD_MODE" == "docker" && -n "$DOCKER_PKG_BUILD" && ! -x "$DOCKER_PKG_BUILD" ]] && { log_error "Not executable: $DOCKER_PKG_BUILD"; exit 1; } # Handle local source if [[ -n "$LOCAL_SOURCE" ]]; then @@ -165,39 +201,39 @@ if [[ -n "$LOCAL_SOURCE" ]]; then fi log_step "Configuration:" -[[ -n "$LOCAL_SOURCE" ]] && log_info " Source: local ($KERNEL_DIR)" \ - || log_info " Repo: $REPO branch: $BRANCH" +if [[ -n "$DSC" ]]; then + log_info " Source: $DSC" +elif [[ -n "$LOCAL_SOURCE" ]]; then + log_info " Source: local ($KERNEL_DIR)" +else + log_info " Repo: $REPO branch: $BRANCH" +fi log_info " Output: $OUTPUT_DIR" +[[ "$SOURCE_PACKAGE" == true ]] && log_info " Building: source package only" log_info " Distro: $DISTRO mode: $BUILD_MODE" -[[ "$BUILD_MODE" == "docker" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" -[[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" -[[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" +[[ "$BUILD_MODE" == "docker" && -n "$DOCKER_PKG_BUILD" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" [[ -n "$PROFILES" ]] && log_info " Profiles: $PROFILES" -[[ -n "$ENABLE_CONFIGS" ]] && log_info " Extra configs: $ENABLE_CONFIGS" -[[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" -[[ "$SKIP_PREPARE" == true ]] && log_info " Skip prepare: yes (source already prepared by prepare-source.sh)" +# The rest describes preparing a tree, which a .dsc has already had done. +if [[ -z "$DSC" ]]; then + [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" \ + || log_info " Flavour: $FLAVOUR" + [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" + [[ -n "$ENABLE_CONFIGS" ]] && log_info " Extra configs: $ENABLE_CONFIGS" + [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" + [[ "$SKIP_PREPARE" == true ]] && log_info " Skip prepare: yes (source already prepared by prepare-source.sh)" +fi echo -# ── Helper: derive LOCALVERSION from a tag name ────────────────────────────── -# qcom-next-6.19-rc8-20260210 → qcom-next-20260210 -_auto_localversion() { - local tag="$1" - if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$ ]]; then - echo "${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" - else - echo "$tag" - fi -} - # ── Git operations: resolve ref → sync → checkout ──────────────────────────── -if [[ -z "$LOCAL_SOURCE" ]]; then - # Resolve the latest tag remotely before any network I/O (avoids fetching all tags) +if [[ -n "$DSC" ]]; then + : # No tree: the source package is the source. +elif [[ -z "$LOCAL_SOURCE" ]]; then + # Resolve the latest tag remotely before any network I/O (avoids fetching + # all tags). The same script CI uses, so "latest" means the same thing + # here: the newest trailing date, not the highest kernel version. if [[ "$LATEST_TAG" == true ]]; then log_step "Finding latest qcom-next-* tag from remote..." - TAG=$(git ls-remote --tags "$REPO" 'refs/tags/qcom-next-*' \ - | awk '{print $2}' | sed 's|refs/tags/||' | grep -v '\^{}' \ - | sort -V | tail -1) - [[ -n "$TAG" ]] || { log_error "No qcom-next-* tags found in $REPO"; exit 1; } + TAG=$("$SCRIPT_DIR/ci/scripts/resolve-kernel-ref.sh" --url "$REPO" --latest-tag 'qcom-next-*') log_info "Latest tag: $TAG" fi @@ -229,39 +265,25 @@ if [[ -z "$LOCAL_SOURCE" ]]; then git -C "$KERNEL_DIR" checkout -B "$BRANCH" FETCH_HEAD fi fi - - # Auto-detect LOCALVERSION from tag (applies to both clone and update paths) - if [[ -n "$TAG" && -z "$LOCALVERSION" ]]; then - LOCALVERSION="$(_auto_localversion "$TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION'" - fi -fi - -cd "$KERNEL_DIR" - -# ── Local source: LOCALVERSION detection ───────────────────────────────────── -if [[ -n "$LOCAL_SOURCE" ]]; then +else log_info "Using local source as-is (skipping git checkout)" - if [[ -z "$LOCALVERSION" ]]; then - GIT_TAG=$(git describe --tags --exact-match 2>/dev/null || true) - if [[ -n "$GIT_TAG" ]]; then - LOCALVERSION="$(_auto_localversion "$GIT_TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' from tag '$GIT_TAG'" - else - log_warn "LOCALVERSION not set and no exact git tag found." - log_warn "Package will be named linux-image--qcom (no branch/ABI suffix)." - log_warn "Use --localversion to specify, e.g.: --localversion qcom-next-20260312" - fi - fi fi +[[ -n "$DSC" ]] || cd "$KERNEL_DIR" + # ── Source preparation ──────────────────────────────────────────────────────── # Delegates to prepare-source.sh, which is the single source of truth for -# debian/ injection, config fragment activation, and debian/rules prepare. +# debian/ injection, config fragment activation, version derivation, and +# debian/rules prepare. Nothing about the version is decided here: an explicit +# --localversion is passed through, and otherwise prepare-source.sh derives +# it from the checkout the way CI does. # Skipped when --skip-prepare is set (CI mode: prepare-source.sh already ran -# as a dedicated prior step). -if [[ "$SKIP_PREPARE" != true ]]; then - PREPARE_ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO" --debian-dir "$DEBIAN_DIR") +# as a dedicated prior step), and when there is no tree to prepare. +if [[ -n "$DSC" ]]; then + : +elif [[ "$SKIP_PREPARE" != true ]]; then + PREPARE_ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO" --debian-dir "$DEBIAN_DIR" + --flavour "$FLAVOUR") [[ -n "$LOCALVERSION" ]] && PREPARE_ARGS+=(--localversion "$LOCALVERSION") [[ -n "$KVER_EXTRA" ]] && PREPARE_ARGS+=(--kver-extra "$KVER_EXTRA") [[ -n "$ENABLE_CONFIGS" ]] && PREPARE_ARGS+=(--kernel-config "$ENABLE_CONFIGS") @@ -279,12 +301,26 @@ else } fi -# ── Build ──────────────────────────────────────────────────────────────────── +# ── Source package ─────────────────────────────────────────────────────────── mkdir -p "$OUTPUT_DIR" +if [[ "$SOURCE_PACKAGE" == true ]]; then + log_step "Building the source package into $OUTPUT_DIR..." + "$SCRIPT_DIR/build-source-package.sh" --source-dir "$KERNEL_DIR" --output-dir "$OUTPUT_DIR" + echo + log_info "Build binaries from it with:" + log_info " $0 --dsc $OUTPUT_DIR/$(cd "$OUTPUT_DIR" && ls -- *.dsc | tail -1) --distro $DISTRO" + exit 0 +fi + +# ── Build ──────────────────────────────────────────────────────────────────── log_step "Building kernel package (mode: $BUILD_MODE)..." [[ -n "$PROFILES" ]] && log_info "Build profiles: $PROFILES" echo +# Every mode builds either the prepared tree or the source package. The +# source-package path is the one CI takes for every suite, so it is the one to +# use when a local build must produce what CI produced: same .dsc in, same +# binaries out. case "$BUILD_MODE" in docker) USE_SUDO="" @@ -294,26 +330,70 @@ case "$BUILD_MODE" in [[ $REPLY =~ ^[Yy]$ ]] || { log_error "Aborted."; exit 1; } USE_SUDO="sudo" } - BUILD_CMD=("$DOCKER_PKG_BUILD" - --skip-gbp - --no-update-check - --source-dir "$KERNEL_DIR" - --output-dir "$OUTPUT_DIR" - --distro "$DISTRO") - [[ -n "$PROFILES" ]] && BUILD_CMD+=(--profiles "$PROFILES") - ${USE_SUDO:+sudo} "${BUILD_CMD[@]}" + if [[ -n "$DSC" ]]; then + # docker_deb_build.py builds from a tree only, so the .dsc is + # handed to sbuild inside the same builder image directly, with + # the same sbuild flags docker_deb_build.py uses. The image is + # named as docker-pkg-build names it; when it is missing, + # docker_deb_build.py --rebuild builds it from its Dockerfile. + DOCKER_IMAGE="ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" + ${USE_SUDO:+sudo} docker image inspect "$DOCKER_IMAGE" >/dev/null 2>&1 || { + [[ -n "$DOCKER_PKG_BUILD" ]] || { + log_error "Builder image $DOCKER_IMAGE is not present, and docker_deb_build.py was not found to build it." + log_error "Pull the image, or point --docker-build at a docker-pkg-build checkout." + exit 1 + } + log_info "Builder image $DOCKER_IMAGE not present; building it with docker-pkg-build..." + ${USE_SUDO:+sudo} "$DOCKER_PKG_BUILD" --no-update-check --rebuild -d "$DISTRO" + } + # The .dsc's directory is mounted read-only so sbuild can read the + # tarballs it names; results go to the output mount. + SBUILD_CMD="sbuild --chroot-mode=unshare --build-dep-resolver=aptitude" + SBUILD_CMD+=" --no-clean-source --build-dir=/workspace/output" + SBUILD_CMD+=" --host=arm64 --build=arm64 --dist=$DISTRO --no-run-lintian" + [[ -n "$PROFILES" ]] && SBUILD_CMD+=" --profiles=$PROFILES" + SBUILD_CMD+=" /workspace/source/$(basename "$DSC")" + ${USE_SUDO:+sudo} docker run --rm --privileged \ + -v "$(dirname "$DSC"):/workspace/source:ro" \ + -v "$OUTPUT_DIR:/workspace/output:Z" \ + -w /workspace/output \ + "$DOCKER_IMAGE" bash -c "$SBUILD_CMD" + else + BUILD_CMD=("$DOCKER_PKG_BUILD" + --skip-gbp + --no-update-check + --source-dir "$KERNEL_DIR" + --output-dir "$OUTPUT_DIR" + --distro "$DISTRO") + [[ -n "$PROFILES" ]] && BUILD_CMD+=(--profiles "$PROFILES") + ${USE_SUDO:+sudo} "${BUILD_CMD[@]}" + fi ;; native) log_info "Running dpkg-buildpackage on host..." [[ -n "$PROFILES" ]] && export DEB_BUILD_PROFILES="$PROFILES" - dpkg-buildpackage -us -uc -b - find "$(dirname "$KERNEL_DIR")" -maxdepth 1 -name "*.deb" -exec mv -v {} "$OUTPUT_DIR/" \; + if [[ -n "$DSC" ]]; then + # Unpack into the output directory, so dpkg-buildpackage's ../ + # is the output directory and the .deb files land there. + BUILD_TREE="$OUTPUT_DIR/$(basename "$DSC" .dsc)" + rm -rf "$BUILD_TREE" + dpkg-source -x "$DSC" "$BUILD_TREE" + (cd "$BUILD_TREE" && dpkg-buildpackage -us -uc -b) + else + dpkg-buildpackage -us -uc -b + find "$(dirname "$KERNEL_DIR")" -maxdepth 1 -name "*.deb" -exec mv -v {} "$OUTPUT_DIR/" \; + fi ;; sbuild) log_info "Running sbuild for $DISTRO..." - SBUILD_CMD=(sbuild --dist "$DISTRO" --arch arm64 --no-source) + SBUILD_CMD=(sbuild --dist "$DISTRO" --arch arm64) [[ -n "$PROFILES" ]] && SBUILD_CMD+=(--profiles "$PROFILES") - "${SBUILD_CMD[@]}" + if [[ -n "$DSC" ]]; then + # sbuild takes a .dsc directly and writes beside its cwd. + (cd "$OUTPUT_DIR" && "${SBUILD_CMD[@]}" "$DSC") + else + "${SBUILD_CMD[@]}" --no-source + fi ;; esac diff --git a/build-source-package.sh b/build-source-package.sh new file mode 100755 index 00000000..12ba8a0a --- /dev/null +++ b/build-source-package.sh @@ -0,0 +1,236 @@ +#!/bin/bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +set -euo pipefail + +# Build a Debian source package from a prepared kernel tree. +# +# Takes a kernel checkout that prepare-source.sh has already injected debian/ +# into, and produces in an output directory the .orig.tar.gz, .debian.tar.xz, +# .dsc and .changes that a binary build -- local, Debusine, or sbuild in a +# container -- starts from. +# +# The orig tarball is a function of the commit and nothing else. It is written +# with git archive, which sets every entry's mtime to the commit time and its +# owner to root, and compressed with gzip -n, which writes no timestamp. Two +# runs on one commit therefore produce one tarball, byte for byte, and two +# matrix entries that share a source package and upstream version -- trixie +# and forky, differing only in Debian revision -- share one orig, which is +# what an archive holding both requires. +# +# The upstream version names the commit (the ~g field), so the archived +# commit is checked against it: an orig named for one commit must hold that +# commit's tree. The tree is also checked against the commit before the +# package is built, because dpkg-source would otherwise fold any stray file +# into an automatic patch and the package would no longer describe the +# commit its version names. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $*"; } + +usage() { + cat </dev/null || { log_error "$tool not found; install dpkg-dev and git"; exit 1; } +done + +git -C "$SOURCE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { + log_error "$SOURCE_DIR is not a git checkout; the orig tarball is archived from a commit." + exit 1 +} + +# ── Read the package identity from the changelog ───────────────────────────── +# The changelog is what prepare-source.sh generated and what the .dsc will be +# built from, so it is the only source of the name and version here. The sed +# filters keep the values to what Debian policy allows in a package name and +# version, as debusine-action does, so they are safe in filenames. +PKG=$(dpkg-parsechangelog -l "$SOURCE_DIR/debian/changelog" -SSource | sed 's/[^a-z0-9.+-]//g') +VER=$(dpkg-parsechangelog -l "$SOURCE_DIR/debian/changelog" -SVersion | sed 's/[^A-Za-z0-9.+~:-]//g') +[[ -n "$PKG" && -n "$VER" ]] || { log_error "Could not read Source and Version from debian/changelog"; exit 1; } +UPSTREAM_VER="${VER%-*}" +[[ "$UPSTREAM_VER" != "$VER" ]] || { + log_error "Version $VER has no Debian revision; a 3.0 (quilt) package needs one." + exit 1 +} + +# ── Resolve the commit and check it against the version ────────────────────── +[[ -n "$COMMIT" ]] || COMMIT=HEAD +COMMIT_SHA=$(git -C "$SOURCE_DIR" rev-parse --verify "${COMMIT}^{commit}") || { + log_error "Not a commit in $SOURCE_DIR: $COMMIT" + exit 1 +} + +# debian/rules spells the commit into the upstream version as ~g<12 hex>. An +# orig named for that commit must hold that commit's tree, so the two are +# checked against each other rather than trusted to agree. A version without +# the field (an explicit --localversion with no snapshot) names no commit, in +# which case there is nothing to check and the tarball is only as +# reproducible as the caller's choice of commit. +if [[ "$UPSTREAM_VER" =~ ~g([0-9a-f]{12}) ]]; then + VERSION_SHA="${BASH_REMATCH[1]}" + [[ "$COMMIT_SHA" == "$VERSION_SHA"* ]] || { + log_error "Version $VER names commit $VERSION_SHA, but $COMMIT is $COMMIT_SHA." + log_error "Re-run prepare-source.sh on this checkout, or pass --commit $VERSION_SHA." + exit 1 + } +else + log_warn "Version $VER names no commit; the orig tarball is archived from $COMMIT_SHA." +fi + +# ── Check the tree against the commit ──────────────────────────────────────── +# dpkg-source compares the tree with the orig, so anything outside debian/ +# that differs from the commit -- an in-tree build's objects, an edit not +# committed -- would become an automatic patch or abort the build. Refuse it +# here with the list and the command that cleans it, rather than let +# dpkg-source report it one file at a time. +log_step "Checking $SOURCE_DIR matches $COMMIT_SHA outside debian/..." +if [[ "$COMMIT_SHA" != "$(git -C "$SOURCE_DIR" rev-parse HEAD)" ]]; then + log_error "HEAD of $SOURCE_DIR is not $COMMIT; check out the commit the package names." + exit 1 +fi +STRAY=$(git -C "$SOURCE_DIR" status --porcelain --ignored -- . ':(exclude)debian' | head -20) +[[ -z "$STRAY" ]] || { + log_error "$SOURCE_DIR differs from commit $COMMIT_SHA outside debian/:" + echo "$STRAY" + log_error "The source package must describe the commit its version names." + log_error "Clean the tree first, keeping the injected packaging:" + log_error " git -C $SOURCE_DIR checkout -- . && git -C $SOURCE_DIR clean -xdf -- . ':(exclude)debian'" + exit 1 +} + +# ── Write the orig tarball ─────────────────────────────────────────────────── +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)" +ORIG="$OUTPUT_DIR/${PKG}_${UPSTREAM_VER}.orig.tar.gz" +DSC="$OUTPUT_DIR/${PKG}_${VER}.dsc" +CHANGES="$OUTPUT_DIR/${PKG}_${VER}_source.changes" + +PREVIOUS_ORIG_SHA256="" +[[ -f "$ORIG" ]] && PREVIOUS_ORIG_SHA256=$(sha256sum "$ORIG" | cut -d' ' -f1) + +log_step "Writing $ORIG from commit $COMMIT_SHA..." +# --format=tar piped through gzip rather than --format=tar.gz: git's own +# gzip is also deterministic, but this spells out which compressor decides +# the bytes. The bytes depend on the gzip implementation, so every build +# that must agree on a checksum runs this in the same builder image. +git -C "$SOURCE_DIR" archive --format=tar --prefix="${PKG}-${UPSTREAM_VER}/" "$COMMIT_SHA" \ + | gzip -9n > "$ORIG.tmp" +mv "$ORIG.tmp" "$ORIG" +ORIG_SHA256=$(sha256sum "$ORIG" | cut -d' ' -f1) +log_info " sha256: $ORIG_SHA256" + +if [[ -n "$PREVIOUS_ORIG_SHA256" && "$PREVIOUS_ORIG_SHA256" != "$ORIG_SHA256" ]]; then + # The file that was there carried the same name and so claimed the same + # commit. It cannot have been the same tarball, so something about how + # it was made differs from this run -- which is what this script exists + # to rule out. + log_error "An orig tarball for $PKG $UPSTREAM_VER already existed with sha256 $PREVIOUS_ORIG_SHA256" + log_error "and rebuilding it from $COMMIT_SHA gave $ORIG_SHA256." + log_error "Two files with this name must be one file; find what changed between the two builds." + exit 1 +fi + +# ── Build the .dsc and .changes ────────────────────────────────────────────── +# dpkg-source -b writes into the current directory and finds the orig there, +# so it runs from the output directory. dpkg-genchanges reads debian/ from +# the source tree and is told where the files are and where to write. +log_step "Building the source package..." +rm -f "$DSC" "$OUTPUT_DIR/${PKG}_${VER}.debian.tar."* "$CHANGES" +(cd "$OUTPUT_DIR" && dpkg-source -b "$SOURCE_DIR") +(cd "$SOURCE_DIR" && dpkg-genchanges -S -sa -u"$OUTPUT_DIR" -O"$CHANGES" >/dev/null) + +[[ -f "$DSC" && -f "$CHANGES" ]] || { + log_error "Expected $DSC and $CHANGES to exist after the build" + exit 1 +} + +echo +log_step "Source package complete." +log_info " $CHANGES" +grep -A100 '^Checksums-Sha256:' "$CHANGES" | sed -n '2,/^[A-Z]/{/^ /p}' | sed 's/^/ /' + +if [[ -n "$FIELDS_FILE" ]]; then + cat > "$FIELDS_FILE" <-dkms packages the +# target archive carries: the Debian suites bundle camx and iris-vpu, which +# Ubuntu does not package. +# +# Every entry under "builds" is built nightly. Where such a build is published +# is not stated on it: the workflow that runs the build decides it, because it +# depends on why the build is running rather than on what is being built. The +# nightly promotes its Debian entries into the archive; a PR build of the same +# entry promotes nowhere. So a build entry describes a kernel, and never a +# destination. +# +# "releases" is the exception, and the one place a destination is written down. +# A release is a decision rather than a schedule: nothing runs these entries +# until someone dispatches release.yml, and what they name is an immutable ref +# rather than whatever the branch has reached. Each states the workspace it +# publishes into, because that is the whole of what distinguishes it from the +# nightly build of the same flavour -- so the ref that ships and the archive it +# ships into are both reviewable in a pull request, rather than typed into a +# dispatch form at the moment of releasing. +# +# The two lists are validated separately and never mix: a name may appear in +# both, and means the same build in each. +# +# debian_revision is stated outright, not derived. It carries no marker for +# where a build is in its life: the version is decided once, when the package is +# built, and the same artifact is what any archive later holds, so there is no +# second version for it to sort against. What the revision does say is where +# the suite belongs relative to the other suites -- "~bpo13+1" puts a trixie +# backport below a forky build of the same kernel (see the ordering discussion +# in README.md). No two entries may produce the same srcpkg at the same +# debian_revision. + +builds: + # --------------------------------------------------------------------------- + # qcom-next — the standard kernel. + # --------------------------------------------------------------------------- + # - name: qcom-next-trixie + # suite: trixie + # flavour: qcom-next + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next + # ref_strategy: latest_tag + # tag_pattern: 'qcom-next-*' + # srcpkg: linux-qcom-next + # binpkg: linux-image-qcom-next + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' + + - name: qcom-next-forky + suite: forky + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + # TODO: added manual revision + debian_revision: '0qli1+staging1' + + - name: qcom-next-resolute + suite: resolute + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. + dkms: + - kgsl + debian_revision: '0qli1~26.04.1' + + # # --------------------------------------------------------------------------- + # # qcom-next-debug — the same kernel ref built with the in-tree debug + # # fragments, packaged separately so it installs alongside qcom-next. + # # --------------------------------------------------------------------------- + # - name: qcom-next-debug-trixie + # suite: trixie + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next + # ref_strategy: latest_tag + # tag_pattern: 'qcom-next-*' + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' + + # - name: qcom-next-debug-forky + # suite: forky + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next + # ref_strategy: latest_tag + # tag_pattern: 'qcom-next-*' + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1' + + # # --------------------------------------------------------------------------- + # # qcom-arduino — the Arduino hardware-enablement topic branch. + # # + # # A topic branch, built for early warning of what it will take to carry. + # # --------------------------------------------------------------------------- + # - name: qcom-arduino-trixie + # suite: trixie + # flavour: qcom-arduino + # git_clone: https://github.com/qualcomm-linux/kernel-topics + # branch_or_tag: early/hwe/arduino + # ref_strategy: branch_tip + # srcpkg: linux-qcom-arduino + # binpkg: linux-image-qcom-arduino + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # debian_revision: '0qli1~bpo13+1' + + # - name: qcom-arduino-forky + # suite: forky + # flavour: qcom-arduino + # git_clone: https://github.com/qualcomm-linux/kernel-topics + # branch_or_tag: early/hwe/arduino + # ref_strategy: branch_tip + # srcpkg: linux-qcom-arduino + # binpkg: linux-image-qcom-arduino + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # debian_revision: '0qli1' + + # # --------------------------------------------------------------------------- + # # mainline — Linus's tree, tracked for early warning of upstream breakage. + # # + # # No DKMS: the out-of-tree modules target the Qualcomm tree, and a listed + # # module is a presence contract that would fail the build here. + # # --------------------------------------------------------------------------- + # - name: mainline-trixie + # suite: trixie + # flavour: mainline + # git_clone: https://github.com/torvalds/linux + # branch_or_tag: master + # ref_strategy: branch_tip + # srcpkg: linux-mainline + # binpkg: linux-image-mainline + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1~bpo13+1' + + # - name: mainline-forky + # suite: forky + # flavour: mainline + # git_clone: https://github.com/torvalds/linux + # branch_or_tag: master + # ref_strategy: branch_tip + # srcpkg: linux-mainline + # binpkg: linux-image-mainline + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1' + + # # --------------------------------------------------------------------------- + # # next — linux-next, the integration tree, one step ahead of mainline. + # # + # # next-YYYYMMDD tags are cut daily, so latest_tag resolves the newest of + # # them the same way qcom-next-* does. No DKMS, as above. + # # --------------------------------------------------------------------------- + # - name: next-trixie + # suite: trixie + # flavour: next + # git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + # branch_or_tag: master + # ref_strategy: latest_tag + # tag_pattern: 'next-*' + # srcpkg: linux-next + # binpkg: linux-image-next + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1~bpo13+1' + + # - name: next-forky + # suite: forky + # flavour: next + # git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + # branch_or_tag: master + # ref_strategy: latest_tag + # tag_pattern: 'next-*' + # srcpkg: linux-next + # binpkg: linux-image-next + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1' + +# Releases. Dispatched from release.yml and never scheduled, so nothing here +# runs until someone asks for it by name. +# +# Every entry pins its ref: ref_strategy must be pinned_ref, because a release +# names the exact source that ships. latest_tag would make two dispatches of +# one entry release different kernels, and branch_tip would make them release +# whatever the branch had reached. +# +# Debian suites only. Promotion runs through Debusine, and the Ubuntu path +# publishes to S3 without a workspace to promote into, so an Ubuntu entry here +# would name a destination nothing could deliver it to. +# +# Updating a release means editing branch_or_tag in a pull request, which is +# what puts the ref that ships under review. +releases: + # --------------------------------------------------------------------------- + # qcom-next 7.2 — the standard kernel. + # --------------------------------------------------------------------------- + # - name: qcom-next-trixie + # suite: trixie + # flavour: qcom-next + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next + # binpkg: linux-image-qcom-next + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' + # target_workspace: qli + + # - name: qcom-next-forky + # suite: forky + # flavour: qcom-next + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next + # binpkg: linux-image-qcom-next + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1' + # target_workspace: qli + + # # --------------------------------------------------------------------------- + # # qcom-next-debug 7.2 — the same ref with the in-tree debug fragments. + # # --------------------------------------------------------------------------- + # - name: qcom-next-debug-trixie + # suite: trixie + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' + # target_workspace: qli + + # - name: qcom-next-debug-forky + # suite: forky + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1' + # target_workspace: qli diff --git a/ci/scripts/derive-debian-revision.sh b/ci/scripts/derive-debian-revision.sh deleted file mode 100755 index 71b3af8a..00000000 --- a/ci/scripts/derive-debian-revision.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause-Clear -set -euo pipefail - -# Derive the suite-specific Debian revision for one delivery leg. -# -# Formula: -# debian_revision = stub + suite_suffix_mapping[suite] + delivery_suffix -# delivery_suffix: Daily -> "~", Release -> "" -# -# This is the single implementation of the formula. It is called both by -# resolve-matrix.sh (once per flattened Daily/Release leg) and by -# build-kernel-deb.yml's direct-dispatch path (one suite, no full matrix -# context), so the derivation and its validation live in exactly one place. -# -# Usage: -# ci/scripts/derive-debian-revision.sh --stub 0qli --suite trixie --delivery-type Daily -# ci/scripts/derive-debian-revision.sh --stub 0qli --suite forky --delivery-type Release --matrix-file ci/build-matrix.json -# -# Options: -# --stub STUB Debian version stub. Must be non-empty and must not -# end in ~ (the delivery suffix supplies any -# trailing ~). Required. -# --suite SUITE Target suite; must have an entry in -# suite_suffix_mapping. Required. -# --delivery-type TYPE Daily or Release. Required. -# --matrix-file FILE Path to the matrix JSON containing -# suite_suffix_mapping -# (default: ci/build-matrix.json relative to CWD). -# -# Output: -# Final Debian revision printed to stdout. -# -# Exit codes: -# 0 Success. -# 1 Error (invalid args, malformed or missing suite_suffix_mapping, -# unmapped suite, unsupported delivery type). - -STUB="" -SUITE="" -DELIVERY_TYPE="" -MATRIX_FILE="ci/build-matrix.json" - -usage() { - sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' - exit 1 -} - -while [[ $# -gt 0 ]]; do - case $1 in - --stub) STUB="$2"; shift 2 ;; - --suite) SUITE="$2"; shift 2 ;; - --delivery-type) DELIVERY_TYPE="$2"; shift 2 ;; - --matrix-file) MATRIX_FILE="$2"; shift 2 ;; - -h|--help) usage ;; - *) echo "ERROR: Unknown option: $1" >&2; usage ;; - esac -done - -[[ -n "$STUB" ]] || { echo "ERROR: --stub is required" >&2; exit 1; } -[[ -n "$SUITE" ]] || { echo "ERROR: --suite is required" >&2; exit 1; } -[[ -n "$DELIVERY_TYPE" ]] || { echo "ERROR: --delivery-type is required" >&2; exit 1; } -[[ "$STUB" != *"~" ]] || { echo "ERROR: --stub must not end in ~ (got '$STUB')" >&2; exit 1; } -[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } - -jq empty "$MATRIX_FILE" 2>/dev/null \ - || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } - -mapping_errors=$(jq -r ' - .suite_suffix_mapping as $mapping | - if ($mapping | type) != "object" - then "suite_suffix_mapping is missing or not an object" - else - ( - [$mapping | to_entries[] | select(.value | type != "string") | "suite_suffix_mapping[" + .key + "] must be a string"] - + - [ - $mapping - | to_entries[] - | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) - | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" - ] - + - [ - $mapping - | to_entries - | group_by(.value) - | map(select(length > 1)) - | .[]? - | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" - ] - ) | .[] - end -' "$MATRIX_FILE") - -if [[ -n "$mapping_errors" ]]; then - echo "ERROR: Invalid suite_suffix_mapping in $MATRIX_FILE:" >&2 - while IFS= read -r error; do - [[ -n "$error" ]] && echo " - $error" >&2 - done <<< "$mapping_errors" - exit 1 -fi - -SUFFIX=$(jq -r --arg suite "$SUITE" '.suite_suffix_mapping[$suite] // "__MISSING__"' "$MATRIX_FILE") -[[ "$SUFFIX" != "__MISSING__" ]] || { - echo "ERROR: no suite_suffix_mapping entry for suite '$SUITE'" >&2 - exit 1 -} - -case "$DELIVERY_TYPE" in - Daily) DELIVERY_SUFFIX="~" ;; - Release) DELIVERY_SUFFIX="" ;; - *) - echo "ERROR: --delivery-type must be Daily or Release (got '$DELIVERY_TYPE')" >&2 - exit 1 - ;; -esac - -echo "${STUB}${SUFFIX}${DELIVERY_SUFFIX}" diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index d36e9b0b..0517e10e 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -3,37 +3,99 @@ # SPDX-License-Identifier: BSD-3-Clause-Clear set -euo pipefail -# Derive the LOCALVERSION suffix from a kernel variant and resolved ref. +# Derive the version fields for a build from a kernel flavour and resolved ref. # -# For dated tag builds (ref ends in -YYYYMMDD): -# Produces --. -# Example: qcom-next-7.2-rc3-20260722 -> -qcom-next-20260722 +# The flavour is the kernel's own identity, the part of the kernel release +# that distinguishes two kernels built from the same ref with different +# configuration, so that their linux-image packages install alongside each +# other. It is not the CI identifier for the build: a flavour is built for +# several suites, and all of those builds produce the same kernel release. +# +# Emits LOCALVERSION (the kernel release suffix), SNAPSHOT (the dated component +# of the Debian version) and GITSHA, all derived from the ref in one place. +# SNAPSHOT and GITSHA are emitted alongside rather than recovered from +# LOCALVERSION later: reading them back out means guessing where each field ends +# in a string that also carries a flavour name, and a hex SHA can end in eight +# digits of its own. +# +# For dated tag builds (ref ends in -YYYYMMDD, optionally .): +# Produces +-[.]-g<12 hex>. +# Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g07f50dc44edd +# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g07f50dc44edd +# +# The respin ordinal distinguishes a second tag cut on the same day. It is +# carried verbatim rather than normalised, so the first tag of a day stays +# plain +-: systemd compares the separator before the chunk +# behind it, so an absent ordinal already sorts below a present one and no +# build has to spell a ".0". +# +# The SHA names the commit the tag pointed at when the build was cut, so a +# moved tag cannot silently produce two different kernels under one release. # # For branch-tip builds (ref does not end in a date): -# Uses the kernel variant and a short SHA for uniqueness. -# Example: qcom-next @ 07f50dc44edd -> -qcom-next-g07f50dc44edd -# --sha is required for branch-tip builds. +# Takes the date from the HEAD commit instead of the tag, so the result has +# the same shape as a tag build and orders in the same sequence. +# Example: qcom-next @ 07f50dc44edd, committed 2026-09-04 +# -> +qcom-next-20260904-g07f50dc44edd +# --date is required for these; pass YYYYMMDD.N to separate two branch-tip +# builds sharing a commit date. +# +# The caller supplies the COMMIT date rather than the build date, so that +# rebuilding a commit reproduces its version instead of inventing a higher +# one, and so that the date describes the source rather than when CI ran. It +# also lands in the same space as upstream's tag dates, which track the +# commit each tag is cut from. +# +# The trade-off: a build date always advances, a commit date need not. If the +# branch is ever rewound to an older commit, the next build's version goes +# DOWN and apt will not offer it as an upgrade. That is arguably honest +# -- older source, older version -- but it is the one case where dating by +# the clock would behave differently. +# +# Why the leading '+' and not '-': +# The suffix ends up in KERNELRELEASE (uname -r), which is the 'version' field +# systemd-boot sorts BLS entries on. systemd compares the separator before the +# chunk behind it, and '-' < '+', so joining with '+' puts every -rcN release +# candidate BELOW the final release that follows it: +# +# 7.2.0-rc7+qcom-next-20260821 < 7.2.0+qcom-next-20260826 +# +# Joining with '-' instead falls through to a plain strcmp of "rc" against +# "qcom", where 'r' > 'q', and every rc outranks its own final release in the +# boot menu. This is the same trick Debian's own kernels rely on +# (linux-image-7.1.10+deb14-amd64). It does not affect the Debian version, +# which spells the release candidate ~rcN and orders correctly either way. # # Usage: -# ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 -# ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd +# ci/scripts/derive-localversion.sh --flavour qcom-next --ref qcom-next-7.2-rc3-20260722 --sha 07f50dc44edd +# ci/scripts/derive-localversion.sh --flavour arduino --ref main --sha 07f50dc44edd --date 20260904 # # Options: -# --variant VARIANT Kernel variant identifier. Defaults to qcom-next. +# --flavour FLAVOUR Kernel flavour. Defaults to qcom-next. # --ref REF Kernel ref (tag name or branch name). Required. -# --sha SHA Short commit SHA (required for branch-tip builds). +# --sha SHA Commit SHA, truncated to 12 hex characters. Required. +# --date DATE HEAD commit date as YYYYMMDD or YYYYMMDD.N. Required for +# branch-tip builds; ignored for dated tags, which carry +# their own date. # # Output: -# LOCALVERSION suffix printed to stdout (e.g. -qcom-next-20260722). -# Always starts with a dash. +# Three KEY=VALUE lines on stdout, in GITHUB_ENV / 'set -a' form: +# +# LOCALVERSION=+qcom-next-20260722.1-g07f50dc44edd +# SNAPSHOT=20260722.1 +# GITSHA=07f50dc44edd +# +# LOCALVERSION always starts with a plus. Every build carries a snapshot, +# whether it came from the tag or from the HEAD commit. # # Exit codes: # 0 Success. -# 1 Error (invalid args, branch-tip without --sha). +# 1 Error (invalid args, malformed --sha, branch tip without --date). -VARIANT="qcom-next" +FLAVOUR="qcom-next" REF="" SHA="" +DATE="" usage() { sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' @@ -42,32 +104,52 @@ usage() { while [[ $# -gt 0 ]]; do case $1 in - --variant) VARIANT="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --ref) REF="$2"; shift 2 ;; --sha) SHA="$2"; shift 2 ;; + --date) DATE="$2"; shift 2 ;; -h|--help) usage ;; *) echo "ERROR: Unknown option: $1" >&2; usage ;; esac done [[ -n "$REF" ]] || { echo "ERROR: --ref is required" >&2; exit 1; } -[[ "$VARIANT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "ERROR: --variant must use lowercase letters, digits, and internal hyphens" >&2 +[[ "$FLAVOUR" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "ERROR: --flavour must use lowercase letters, digits, and internal hyphens" >&2 + exit 1 +} +# Every build identifies its commit, so --sha is required for all of them, not +# just the branch tips that cannot be identified any other way. +[[ "$SHA" =~ ^[0-9a-f]{12,40}$ ]] || { + echo "ERROR: --sha is required and must be at least 12 lowercase hex characters (got '$SHA')" >&2 exit 1 } +# 12 chars is upstream's own abbreviation width in -g, and short enough +# to keep the kernel release readable in a boot menu. +GITSHA="${SHA:0:12}" -# Dated tags use a trailing YYYYMMDD snapshot. The matrix selects the tag set; -# the variant supplies the stable package identity used in LOCALVERSION. -if [[ "$REF" =~ -([0-9]{8})$ ]]; then - DATE="${BASH_REMATCH[1]}" - echo "-${VARIANT}-${DATE}" +# Dated tags use a trailing YYYYMMDD snapshot, optionally followed by a respin +# ordinal. The matrix selects the tag set; the flavour supplies the stable +# kernel identity used in LOCALVERSION. +if [[ "$REF" =~ -([0-9]{8}(\.[0-9]+)?)$ ]]; then + SNAPSHOT="${BASH_REMATCH[1]}" + LOCALVERSION="+${FLAVOUR}-${SNAPSHOT}-g${GITSHA}" else - # Branch-tip build: need SHA for uniqueness. - [[ -n "$SHA" ]] || { - echo "ERROR: --sha is required for branch-tip builds (ref '$REF' is not a dated tag)" >&2 + # Branch-tip build: the ref carries no date, so the commit date supplies + # one. Without it these builds had no snapshot at all, which put their + # Debian version below every dated build rather than among them. + [[ -n "$DATE" ]] || { + echo "ERROR: --date is required for branch-tip builds (ref '$REF' is not a dated tag)" >&2 + exit 1 + } + [[ "$DATE" =~ ^[0-9]{8}(\.[0-9]+)?$ ]] || { + echo "ERROR: --date must be YYYYMMDD or YYYYMMDD.N (got '$DATE')" >&2 exit 1 } - # Use first 12 chars of SHA for a compact but unambiguous suffix. - SHORT_SHA="${SHA:0:12}" - echo "-${VARIANT}-g${SHORT_SHA}" + SNAPSHOT="$DATE" + LOCALVERSION="+${FLAVOUR}-${SNAPSHOT}-g${GITSHA}" fi + +echo "LOCALVERSION=${LOCALVERSION}" +echo "SNAPSHOT=${SNAPSHOT}" +echo "GITSHA=${GITSHA}" diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py new file mode 100755 index 00000000..b3532c30 --- /dev/null +++ b/ci/scripts/resolve-matrix.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +"""Validate and select entries from the kernel delivery matrix. + +ci/build-matrix.yaml holds one entry per generated package: a single named +build for a single suite. Nothing here expands or derives anything -- the +matrix is already flat, and each entry states its own debian_revision. This +script validates the whole document, selects the entries a caller asked for, +and prints them. + +The document has two lists of entries, both of the same shape. "builds" is what +is built nightly, and describes a kernel rather than a destination: where such +a build is published is decided by the workflow running it, not stated here -- +the nightly promotes its Debian entries into the archive, and a PR build +promotes nowhere. "releases" is the exception, selected with --releases, and +each of its entries names the workspace it publishes into and pins the exact +ref that ships. + +Both lists are validated in full on every invocation, not just the selected +entries, so a typo in an entry nobody selected fails the run that would have +built its siblings rather than lying in wait. They are validated separately and +never mix: a name may appear in both, and means the same build in each. + +Usage: + ci/scripts/resolve-matrix.py + ci/scripts/resolve-matrix.py --build qcom-next-trixie + ci/scripts/resolve-matrix.py --build qcom-next-trixie,qcom-next-forky + ci/scripts/resolve-matrix.py --flavour qcom-next + ci/scripts/resolve-matrix.py --family ubuntu --allow-empty + ci/scripts/resolve-matrix.py --build qcom-next-trixie --field debian_revision + ci/scripts/resolve-matrix.py --releases + ci/scripts/resolve-matrix.py --releases --build qcom-next-trixie + +Options: + --releases Select from the releases list rather than from + builds. Release entries carry a + target_workspace and pin their ref, so this is + how release.yml asks what may be released. + --build NAMES Select only these builds by name, comma-separated. + A name identifies one build, so this is the way + to ask for a specific set of them. + --flavour FLAVOURS Select only these flavours, comma-separated. A + flavour spans its suites, so this asks for one + kernel everywhere it is built. + --suite SUITES Select only these suites, comma-separated. + --family FAMILY Select only the builds taking one build path, + debian or ubuntu. Derived from suite, so it + selects by how a build is built rather than by + naming every suite that is built that way. + --allow-empty Print [] rather than failing when the filters + select nothing. For a caller asking each family + for the same selection, where one of them + having nothing to build is an ordinary answer. + --field NAME Print just this field of the single selected + entry, unquoted. Errors unless exactly one + entry matches. + --matrix-file FILE Matrix path (default: ci/build-matrix.yaml + relative to CWD). + + Filters combine: an entry must match every filter given. Every name in a + filter must match at least one entry, so a typo or a stale name fails + instead of quietly narrowing the build set. That check is per filter, so + --allow-empty still rejects a name that matches nothing. + +Output: + Without --field, a compact JSON array of the selected entries, ready for a + GitHub Actions matrix `include`. Each entry carries a derived family field + naming its build path. kernel_config and dkms are joined into the + comma-separated strings that the build workflows' kernel-config and dkms + inputs, and prepare-source.sh's --kernel-config and --dkms, expect; every + other field is passed through as written. + + With --field, the named field's value alone, so a workflow step can capture + it directly. + +Exit codes: + 0 Success, at least one entry selected. + 1 Error (invalid arguments, matrix validation failure, no matching entry, + or --field matching more than one entry). +""" + +import argparse +import json +import re +import sys +from collections import defaultdict + +try: + import yaml +except ImportError: + sys.exit( + "ERROR: PyYAML is required to read the delivery matrix.\n" + " Install it with 'apt-get install python3-yaml' or 'pip install pyyaml'." + ) + +DEFAULT_MATRIX_FILE = "ci/build-matrix.yaml" + +REF_STRATEGIES = ("latest_tag", "branch_tip", "pinned_ref") + +# Which build path a suite takes. Debian-family suites are built by Debusine; +# everything else is built in a suite-matched docker container on the +# self-hosted runner. The two paths are different workflows with different +# runners, containers and publish steps, so a caller selects entries by family +# and calls the one workflow that builds them -- rather than every build leg +# starting both and skipping one. +DEBIAN_SUITES = ("trixie", "forky", "sid", "unstable") +FAMILIES = ("debian", "ubuntu") + + +def family_for(suite): + """Return the build family a suite belongs to.""" + return "debian" if suite in DEBIAN_SUITES else "ubuntu" + + +REQUIRED_STRING_FIELDS = ( + "name", + "suite", + "flavour", + "git_clone", + "branch_or_tag", + "ref_strategy", + "srcpkg", + "binpkg", + "debian_revision", +) + +OPTIONAL_STRING_FIELDS = ( + "tag_pattern", + "localversion", + "kver_extra", + "debusine_parent_workspace", +) + +# A release entry states its destination, and a build entry may not. This is +# the whole difference between the two lists: where a nightly build is +# published follows from why it is running, so its entry says nothing about it, +# while a release exists precisely to put one ref into one archive. +RELEASE_REQUIRED_STRING_FIELDS = ("target_workspace",) + +KNOWN_FIELDS = frozenset( + REQUIRED_STRING_FIELDS + OPTIONAL_STRING_FIELDS + ("kernel_config", "dkms") +) + +RELEASE_KNOWN_FIELDS = KNOWN_FIELDS | frozenset(RELEASE_REQUIRED_STRING_FIELDS) + +# Everything about what is built is anchored on flavour, not on the build's +# name. The flavour is the kernel's own identity: it becomes the LOCALVERSION +# suffix, so two flavours built from one ref produce distinct kernel releases +# whose linux-image packages install alongside each other. A build's name only +# labels it in CI -- the Actions job, and what a dispatch asks for -- and +# carries no packaging meaning. + +# Fields that identify the flavour itself rather than one of its build legs. +# Every entry for a flavour must agree on them, because they decide what the +# package is; the entries only differ in where it is delivered. +FLAVOUR_IDENTITY_FIELDS = ("srcpkg", "binpkg", "kernel_config") + +# Fields a flavour may vary between suites but not within one. The out-of-tree +# module set depends on which -dkms packages the target archive has, so +# it is a property of the flavour in a suite rather than of the flavour. +SUITE_IDENTITY_FIELDS = ("dkms",) + +# Fields deciding which kernel tree is built. All entries for one flavour build +# the same source, so a stale suite cannot quietly ship a different kernel from +# its siblings. +REF_FIELDS = ("git_clone", "branch_or_tag", "ref_strategy", "tag_pattern") + +NAME_RE = re.compile(r"^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") + +# The dispatch forms of daily.yml and release.yml take one builds field, where +# "all" means every entry and anything else is a list of names. A build +# actually called all would be unreachable through them, so the matrix may not +# define one. +RESERVED_NAMES = ("all",) + +# A Debian revision: no hyphen (that would start a new revision component) and +# none of the characters dpkg rejects in a version. +REVISION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+~]*$") + +# An "intree:" entry names a fragment shipped by the kernel source, as a path +# relative to the kernel source root. A bare entry names a fragment in +# debian/config-available/, with or without its .config extension. +INTREE_PATH_RE = re.compile(r"^([A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\.config$") +BARE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + +# A dkms entry is a package name stem: the build wants -dkms available, +# and generates both the Build-Depends entry and the debian/dkms-modules +# manifest from it. Same shape debian/rules enforces at prepare time, checked +# here so a typo fails before a runner is claimed rather than mid-build. +DKMS_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9+.-]*$") + + +def fragment_basename(fragment): + """Filename a fragment lands under in debian/config/, minus .config. + + Every fragment is copied into debian/config/ under its basename, so + arch/arm64/configs/hardening.config and kernel/configs/hardening.config + collide there even though the entries differ. + """ + return fragment.removeprefix("intree:").rsplit("/", 1)[-1].removesuffix(".config") + + +def check_kernel_config(entry, report): + """Validate one entry's kernel_config list.""" + fragments = entry.get("kernel_config") + if not isinstance(fragments, list): + report("kernel_config must be a list") + return + if any(not isinstance(f, str) or not f for f in fragments): + report("kernel_config must contain only non-empty strings") + return + if any("," in f for f in fragments): + report( + "kernel_config entries must not contain commas; " + "use one list element per fragment" + ) + return + if len(set(fragments)) != len(fragments): + report("kernel_config must not contain duplicates") + + for fragment in fragments: + if fragment.startswith("intree:"): + path = fragment.removeprefix("intree:") + traversal = path == ".." or path.startswith("../") or "/../" in path + if not INTREE_PATH_RE.match(path) or traversal or path.endswith("/.."): + report( + f"intree: entry '{fragment}' must be a kernel-source-relative " + "path ending in .config " + "(e.g. intree:arch/arm64/configs/qcom_debug.config)" + ) + elif not BARE_FRAGMENT_RE.match(fragment): + report( + f"kernel_config entry '{fragment}' must be a fragment name from " + "debian/config-available/ or an intree: path" + ) + + basenames = [fragment_basename(f) for f in fragments] + if len(set(basenames)) != len(basenames): + report("kernel_config entries must not resolve to the same fragment filename") + + +def check_dkms(entry, report): + """Validate one entry's dkms list. + + An empty list is meaningful: it bundles nothing, as opposed to leaving the + field out, which the matrix does not allow. + """ + modules = entry.get("dkms") + if not isinstance(modules, list): + report("dkms must be a list") + return + if any(not isinstance(m, str) or not m for m in modules): + report("dkms must contain only non-empty strings") + return + if len(set(modules)) != len(modules): + report("dkms must not contain duplicates") + + for module in modules: + if module.endswith("-dkms"): + report( + f"dkms entry '{module}' must omit the -dkms suffix " + f"(use '{module.removesuffix('-dkms')}')" + ) + elif not DKMS_NAME_RE.match(module): + report(f"dkms entry '{module}' must be a package name stem, e.g. kgsl") + + +def check_entry(entry, report, release=False): + """Validate one delivery entry in isolation.""" + required = REQUIRED_STRING_FIELDS + known = KNOWN_FIELDS + if release: + required += RELEASE_REQUIRED_STRING_FIELDS + known = RELEASE_KNOWN_FIELDS + + for field in required: + value = entry.get(field) + if not isinstance(value, str) or not value: + report(f"missing or invalid {field}") + + for field in OPTIONAL_STRING_FIELDS: + if field in entry and not isinstance(entry[field], str): + report(f"invalid {field}") + + for field in sorted(set(entry) - known): + # target_workspace on a build entry is the likely version of this, and + # says something the nightly would silently ignore: a build entry can + # no more choose an archive than it can choose to run. + report(f"unknown field {field}") + + for field in ("name", "suite", "flavour"): + value = entry.get(field) + if isinstance(value, str) and not NAME_RE.match(value): + report(f"{field} must use lowercase letters, digits, and internal hyphens") + + if entry.get("name") in RESERVED_NAMES: + report( + f"name {entry['name']} is reserved by the build workflows' dispatch " + "form, where it selects every entry rather than one of them" + ) + + check_kernel_config(entry, report) + check_dkms(entry, report) + + ref_strategy = entry.get("ref_strategy") + if ref_strategy not in REF_STRATEGIES: + report("ref_strategy must be " + ", ".join(REF_STRATEGIES)) + + if ref_strategy == "latest_tag": + if not entry.get("tag_pattern"): + report("missing or invalid tag_pattern") + elif "tag_pattern" in entry: + report("tag_pattern is only valid with ref_strategy=latest_tag") + + revision = entry.get("debian_revision") + if isinstance(revision, str) and revision and not REVISION_RE.match(revision): + report( + f'debian_revision "{revision}" is not a valid Debian revision ' + "(letters, digits, and . + ~ only, starting with a letter or digit)" + ) + + if not release: + return + + # A release names the exact source that ships. latest_tag would make two + # dispatches of one entry release different kernels, and branch_tip would + # make them release whatever the branch had reached by then. + if ref_strategy in REF_STRATEGIES and ref_strategy != "pinned_ref": + report( + f"ref_strategy must be pinned_ref for a release (got {ref_strategy}); " + "a release names one immutable ref" + ) + + # Promotion runs through Debusine, which is the Debian path. The Ubuntu + # path publishes to S3 and has no workspace to promote into, so an entry + # here would name a destination nothing could deliver it to. + suite = entry.get("suite") + if isinstance(suite, str) and suite and family_for(suite) != "debian": + report( + f"suite {suite} is not a Debian suite; only the Debian path can " + "promote into a workspace, so only its builds can be released" + ) + + +def describe(entry, index): + """Label an entry in an error message by what identifies it to a reader.""" + if not isinstance(entry, dict): + return f"entry {index}" + parts = [ + str(entry[field]) + for field in ("name", "suite") + if isinstance(entry.get(field), str) + ] + return f"entry {index} ({'/'.join(parts)})" if parts else f"entry {index}" + + +def check_consistency(builds, errors): + """Validate the invariants that span entries. + + Entries are written out in full, so the matrix can state a flavour twice + and disagree with itself. These checks are what makes that duplication + safe to read at face value. + + Everything about the package is grouped by flavour. A build's name is + only checked for the one thing it is used for: identifying that build + uniquely. + """ + by_name = defaultdict(list) + by_flavour = defaultdict(list) + by_flavour_suite = defaultdict(list) + by_package_version = defaultdict(list) + flavours_by_package = defaultdict(set) + + for entry in builds: + if not isinstance(entry, dict): + continue + flavour = entry.get("flavour") + suite = entry.get("suite") + if not isinstance(flavour, str): + continue + + by_flavour[flavour].append(entry) + by_flavour_suite[(flavour, suite)].append(entry) + by_name[entry.get("name")].append(entry) + + for field in ("srcpkg", "binpkg"): + if isinstance(entry.get(field), str) and entry[field]: + flavours_by_package[(field, entry[field])].add(flavour) + + if isinstance(entry.get("srcpkg"), str) and isinstance( + entry.get("debian_revision"), str + ): + by_package_version[(entry["srcpkg"], entry["debian_revision"])].append(entry) + + # A name identifies one build: it is the Actions job name and what a + # workflow dispatch asks for. Two entries sharing one would give a run two + # identically named jobs and make the dispatch filter ambiguous. + for name, entries in sorted(by_name.items(), key=lambda item: str(item[0])): + if len(entries) > 1: + suites = ", ".join(sorted(str(e.get("suite")) for e in entries)) + errors.append( + f"name {name} is used by {len(entries)} entries " + f"(suites {suites}); a name identifies exactly one build" + ) + + for flavour, entries in sorted(by_flavour.items()): + for field in FLAVOUR_IDENTITY_FIELDS: + values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} + if len(values) > 1: + errors.append( + f"flavour {flavour} must use one {field} across all its " + "entries (got " + ", ".join(sorted(values)) + ")" + ) + + for (flavour, suite), entries in sorted( + by_flavour_suite.items(), key=lambda item: str(item[0]) + ): + for field in SUITE_IDENTITY_FIELDS: + values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} + if len(values) > 1: + errors.append( + f"flavour {flavour} must use one {field} across its " + f"{suite} entries (got " + ", ".join(sorted(values)) + ")" + ) + + for flavour, entries in sorted(by_flavour.items()): + for field in REF_FIELDS: + values = {entry.get(field) for entry in entries} + if len(values) > 1: + rendered = ", ".join(sorted(str(v) for v in values)) + errors.append( + f"flavour {flavour} must build one {field} across its " + f"entries (got {rendered})" + ) + + # Two flavours sharing a package name would overwrite each other in the + # archive; the whole point of a second flavour is a second package. + for (field, package), flavours in sorted(flavours_by_package.items()): + if len(flavours) > 1: + errors.append( + f"{field} {package} is shared by flavours " + ", ".join(sorted(flavours)) + ) + + for (srcpkg, revision), entries in sorted( + by_package_version.items(), key=lambda item: str(item[0]) + ): + if len(entries) > 1: + suites = ", ".join(sorted(str(entry.get("suite")) for entry in entries)) + errors.append( + f"srcpkg {srcpkg} is built at debian_revision {revision} for " + f"suites {suites}; each entry needs a revision of its own" + ) + + +def validate(builds, release=False): + """Return every problem found in one list of entries, as messages.""" + errors = [] + for index, entry in enumerate(builds): + if not isinstance(entry, dict): + errors.append(f"{describe(entry, index)}: delivery entries must be mappings") + continue + label = describe(entry, index) + check_entry( + entry, + lambda message, label=label: errors.append(f"{label}: {message}"), + release=release, + ) + check_consistency(builds, errors) + return errors + + +ROOT_KEYS = ("builds", "releases") + + +def load_matrix(path, section="builds"): + """Read and validate the whole matrix, returning one section's entries. + + Both sections are validated however few of them the caller wants, so a + broken release entry fails a nightly run too -- rather than waiting to be + found by whoever next tries to release. + """ + try: + with open(path, encoding="utf-8") as handle: + document = yaml.safe_load(handle) + except FileNotFoundError: + sys.exit(f"ERROR: Matrix file not found: {path}") + except OSError as error: + sys.exit(f"ERROR: Cannot read {path}: {error}") + except yaml.YAMLError as error: + sys.exit(f"ERROR: Invalid YAML in {path}: {error}") + + if not isinstance(document, dict): + sys.exit(f"ERROR: {path}: matrix root must be a mapping with a builds key") + + # builds and releases are the whole schema. Anything else at the root is a + # leftover from an older matrix (suite_suffix_mapping, say) that would + # otherwise sit there looking authoritative while nothing read it. + unknown_root = sorted(set(document) - set(ROOT_KEYS)) + if unknown_root: + sys.exit( + f"ERROR: {path}: unknown top-level key(s) {', '.join(unknown_root)}; " + + " and ".join(ROOT_KEYS) + " are the only ones" + ) + + sections = {} + for key in ROOT_KEYS: + entries = document.get(key) + # releases is optional, and a matrix with none is a matrix nothing has + # blessed yet rather than a broken one. builds is not: a matrix that + # builds nothing is a mistake in every case. + if entries is None and key != "builds": + sections[key] = [] + continue + if not isinstance(entries, list): + sys.exit(f"ERROR: {path}: {key} must be a list") + if not entries and key == "builds": + sys.exit(f"ERROR: {path}: builds must contain at least one entry") + sections[key] = entries + + errors = [] + for key, entries in sections.items(): + errors += [ + f"{key}: {error}" for error in validate(entries, release=key == "releases") + ] + if errors: + sys.exit( + f"ERROR: Invalid kernel delivery matrix in {path}:\n" + + "\n".join(f" - {error}" for error in errors) + ) + + selected = sections[section] + if not selected: + sys.exit( + f"ERROR: {path}: no {section} are defined, so there is nothing to " + "select from" + ) + + # Derived, never written: family follows from suite, so the matrix cannot + # state one that disagrees with the suite it is built for. Attached after + # validation, which rejects family as an unknown field on an entry. + for entry in selected: + entry["family"] = family_for(entry["suite"]) + + return selected + + +def parse_filter(value): + """Split a comma-separated filter into names, or None when unset.""" + if not value: + return None + return [name.strip() for name in value.split(",") if name.strip()] + + +def select(builds, filters): + """Return the entries matching the requested filters, in matrix order. + + filters maps a field name to the list of values allowed for it, or to + None when that filter was not given. + """ + return [ + entry + for entry in builds + if all( + wanted is None or entry[field] in wanted + for field, wanted in filters.items() + ) + ] + + +def unmatched_filters(builds, filters): + """Names asked for that no entry offers. + + family is exempt: it routes a selection to a build path rather than naming + something in the matrix, argparse already restricts it to a real family, + and a matrix having no builds on one path is an ordinary state of it rather + than a mistake in the request. + """ + missing = [] + for field, wanted in filters.items(): + if wanted is None or field == "family": + continue + available = {entry[field] for entry in builds} + for name in wanted: + if name not in available: + missing.append( + f"no entry has {field} {name} " + f"(available: {', '.join(sorted(available))})" + ) + return missing + + +def describe_selection(filters): + """Render the active filters for an error message.""" + parts = [ + f"{field}={','.join(wanted)}" + for field, wanted in filters.items() + if wanted is not None + ] + return " ".join(parts) if parts else "the whole matrix" + + +def for_workflow(entry): + """Shape one entry the way the build workflows' inputs expect it.""" + return { + **entry, + "kernel_config": ",".join(entry["kernel_config"]), + "dkms": ",".join(entry["dkms"]), + } + + +def main(): + parser = argparse.ArgumentParser( + description="Validate and select entries from the kernel delivery matrix.", + epilog="See the module docstring in this file for full documentation.", + ) + parser.add_argument( + "--releases", + action="store_true", + help="select from the releases list rather than from builds", + ) + parser.add_argument( + "--build", + default="", + help="select only these builds by name, comma-separated", + ) + parser.add_argument( + "--flavour", default="", help="select only these flavours, comma-separated" + ) + parser.add_argument( + "--suite", default="", help="select only these suites, comma-separated" + ) + parser.add_argument( + "--family", + default="", + choices=("",) + FAMILIES, + help="select only the builds taking this build path", + ) + parser.add_argument( + "--allow-empty", + action="store_true", + help="print [] instead of failing when the filters select nothing", + ) + parser.add_argument( + "--field", + default="", + help="print just this field of the single selected entry", + ) + parser.add_argument( + "--matrix-file", default=DEFAULT_MATRIX_FILE, help="path to the matrix YAML" + ) + args = parser.parse_args() + + builds = load_matrix( + args.matrix_file, section="releases" if args.releases else "builds" + ) + filters = { + "name": parse_filter(args.build), + "flavour": parse_filter(args.flavour), + "suite": parse_filter(args.suite), + "family": parse_filter(args.family), + } + what = describe_selection(filters) + + # Report a name that matches nothing before reporting an empty selection: + # "no entry has name qcom-nxt" says what to fix, where "no entries found" + # leaves the reader to work out which filter was wrong. + unmatched = unmatched_filters(builds, filters) + if unmatched: + sys.exit( + f"ERROR: Nothing to build for {what}:\n" + + "\n".join(f" - {problem}" for problem in unmatched) + ) + + selected = select(builds, filters) + if not selected: + # A caller splitting one dispatch across both build paths asks each + # family for the same selection, and one of them legitimately has + # nothing to build. Every name in the request still had to match + # something above, so this is an empty intersection, not a typo. + if args.allow_empty and not args.field: + print("[]") + return + sys.exit(f"ERROR: No matrix entries found for {what}") + + if not args.field: + print(json.dumps([for_workflow(entry) for entry in selected], separators=(",", ":"))) + return + + if len(selected) > 1: + sys.exit( + f"ERROR: --field {args.field} needs exactly one entry, but {what} " + f"selects {len(selected)}; narrow it with --build" + ) + + entry = for_workflow(selected[0]) + if args.field not in entry: + sys.exit( + f"ERROR: {what} has no field {args.field}; " + "available: " + ", ".join(sorted(entry)) + ) + print(entry[args.field]) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/resolve-matrix.sh b/ci/scripts/resolve-matrix.sh deleted file mode 100755 index 4c120641..00000000 --- a/ci/scripts/resolve-matrix.sh +++ /dev/null @@ -1,400 +0,0 @@ -#!/bin/bash -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause-Clear -set -euo pipefail - -# Resolve and flatten the kernel delivery matrix for a given delivery type. -# -# The matrix root is an object with two top-level keys: -# - "suite_suffix_mapping": a suite -> Debian suffix map shared by every -# kernel variant and delivery type (e.g. "trixie": "~bpo13+1"). -# - "deliveries": the matrix rows. Each kernel_variant owns exactly one -# Daily row and one Release row. A row declares every input needed by -# that delivery, including a debian_version_stub. Three fields are -# list-valued: suites, which is expanded into isolated legs; -# kernel_config, which is one config fragment per element; and dkms, -# which is one out-of-tree module per element. A fragment is -# either a bare name from debian/config-available/ or an "intree:" entry -# naming a path relative to the kernel source root -# (e.g. intree:arch/arm64/configs/qcom_debug.config). A dkms entry is the -# stem of a -dkms package, e.g. "kgsl". -# -# Each flattened leg's final debian_revision is derived from -# debian_version_stub, suite_suffix_mapping[suite], and the delivery type via -# ci/scripts/derive-debian-revision.sh, so the formula has exactly one -# implementation shared with build-kernel-deb.yml's direct-dispatch path. Each -# row also carries debian_version_suffix ("~" for Daily, "" for Release) as a -# visible, validated record of that same delivery-type mapping; it is checked -# against the row's type but never fed into derivation, so a copy/paste error -# here fails fast instead of silently drifting from the formula's single -# implementation. -# -# Usage: -# ci/scripts/resolve-matrix.sh --type Daily -# ci/scripts/resolve-matrix.sh --type Release -# ci/scripts/resolve-matrix.sh --type Daily --single-suite trixie -# ci/scripts/resolve-matrix.sh --type Daily --kernel-variant qcom-next -# ci/scripts/resolve-matrix.sh --type Daily --matrix-file path/to/matrix.json -# -# Options: -# --type TYPE Delivery type to filter (Daily or Release). -# Required. -# --single-suite SUITE Emit only entries for this suite. -# --kernel-variant VARIANT Emit only entries for this kernel variant. -# --matrix-file FILE Path to the matrix JSON file -# (default: ci/build-matrix.json relative to CWD). -# -# Output: -# Compact JSON array to stdout. Every entry has a single suite, the -# kernel_variant that scopes its artifacts, Debusine workspace, and logs, -# and a suite-specific debian_revision (debian_version_stub and -# debian_version_suffix are consumed and removed). kernel_config and dkms are -# joined into the comma-separated strings that build-kernel-deb.yml's -# kernel-config and dkms inputs — and in turn prepare-source.sh's -# --kernel-config and --dkms — expect. -# -# Exit codes: -# 0 Success, at least one entry emitted. -# 1 Error (invalid arguments, matrix validation failure, no matching -# entry, revision derivation failure). - -TYPE="" -SINGLE_SUITE="" -KERNEL_VARIANT="" -MATRIX_FILE="ci/build-matrix.json" - -usage() { - sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' - exit 1 -} - -while [[ $# -gt 0 ]]; do - case $1 in - --type) TYPE="$2"; shift 2 ;; - --single-suite) SINGLE_SUITE="$2"; shift 2 ;; - --kernel-variant) KERNEL_VARIANT="$2"; shift 2 ;; - --matrix-file) MATRIX_FILE="$2"; shift 2 ;; - -h|--help) usage ;; - *) echo "ERROR: Unknown option: $1" >&2; usage ;; - esac -done - -[[ "$TYPE" == "Daily" || "$TYPE" == "Release" ]] || { - echo "ERROR: --type must be Daily or Release" >&2 - exit 1 -} -[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } - -jq empty "$MATRIX_FILE" 2>/dev/null \ - || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } - -validation_errors=$(jq -r ' - def required_string($field): - if (has($field) and (.[$field] | type == "string") and (.[$field] | length > 0)) - then empty - else "missing or invalid " + $field - end; - - def optional_string($field): - if (has($field) | not) or (.[$field] | type == "string") - then empty - else "invalid " + $field - end; - - def variant_name_valid: - if (.kernel_variant | type) != "string" - then empty - elif (.kernel_variant | test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$")) - then empty - else "kernel_variant must use lowercase letters, digits, and internal hyphens" - end; - - # An "intree:" entry names a fragment shipped by the kernel source, as a - # path relative to the kernel source root (e.g. - # intree:arch/arm64/configs/qcom_debug.config). Reject absolute paths, - # ".." traversal, and anything not ending in .config so a typo fails here - # rather than deep inside prepare-source.sh. - def intree_path_valid: - .[7:] as $path | - ($path | test("^([A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\\.config$")) - and (($path | test("(^|/)\\.\\.(/|$)")) | not); - - # Every fragment lands in debian/config/ under its basename, so two entries - # sharing one basename (e.g. arch/arm64/configs/hardening.config and - # kernel/configs/hardening.config) collide there. - def fragment_basename: - sub("^intree:"; "") | sub("^.*/"; "") | sub("\\.config$"; ""); - - def kernel_config_valid: - if (.kernel_config | type) != "array" - then "kernel_config must be an array" - elif any(.kernel_config[]; type != "string" or length == 0) - then "kernel_config must contain only non-empty strings" - elif any(.kernel_config[]; test(",")) - then "kernel_config entries must not contain commas; use one array element per fragment" - elif ([.kernel_config[]] | unique | length) != (.kernel_config | length) - then "kernel_config must not contain duplicates" - elif any(.kernel_config[]; startswith("intree:") and (intree_path_valid | not)) - then "intree: entries must be a kernel-source-relative path ending in .config (e.g. intree:arch/arm64/configs/qcom_debug.config)" - elif ([.kernel_config[] | fragment_basename] | unique | length) != (.kernel_config | length) - then "kernel_config entries must not resolve to the same fragment filename" - else empty - end; - - # Out-of-tree modules to build and bundle, each named as the stem of its - # -dkms package. The same shape rules debian/rules prepare enforces, - # applied here so a typo fails before any build job starts rather than in - # the middle of one. - def dkms_valid: - if (.dkms | type) != "array" - then "dkms must be an array" - elif any(.dkms[]; type != "string" or length == 0) - then "dkms must contain only non-empty strings" - elif any(.dkms[]; test(",")) - then "dkms entries must not contain commas; use one array element per module" - elif any(.dkms[]; endswith("-dkms")) - then "dkms entries must omit the -dkms suffix (e.g. kgsl, not kgsl-dkms)" - elif any(.dkms[]; test("^[a-z0-9][a-z0-9+.-]*$") | not) - then "dkms entries must be package name stems (lowercase letters, digits, + . -)" - elif ([.dkms[]] | unique | length) != (.dkms | length) - then "dkms must not contain duplicates" - else empty - end; - - def suites_valid: - if (.suites | type) != "array" or (.suites | length) == 0 - then "suites must be a non-empty array" - elif any(.suites[]; type != "string" or length == 0) - then "suites must contain only non-empty strings" - elif any(.suites[]; test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") | not) - then "suites must use lowercase letters, digits, and internal hyphens" - elif ([.suites[]] | unique | length) != (.suites | length) - then "suites must not contain duplicates" - else empty - end; - - def row_errors($index): - if type != "object" - then "row " + ($index | tostring) + ": matrix entries must be objects" - else - . as $row | - [ - required_string("kernel_variant"), - required_string("type"), - required_string("git_clone"), - required_string("branch_or_tag"), - required_string("ref_strategy"), - required_string("srcpkg"), - required_string("binpkg"), - required_string("debian_version_stub"), - optional_string("debusine_parent_workspace"), - optional_string("localversion"), - optional_string("kver_extra"), - variant_name_valid, - suites_valid, - kernel_config_valid, - dkms_valid, - if (.debian_version_stub | type) == "string" and (.debian_version_stub | test("~$")) - then "debian_version_stub must not end in ~" - else empty end, - if (has("debian_version_suffix") | not) or (.debian_version_suffix | type) != "string" - then "missing or invalid debian_version_suffix" - elif .type == "Daily" and .debian_version_suffix != "~" - then "debian_version_suffix must be \"~\" for Daily rows (got \"" + (.debian_version_suffix | tostring) + "\")" - elif .type == "Release" and .debian_version_suffix != "" - then "debian_version_suffix must be \"\" for Release rows (got \"" + (.debian_version_suffix | tostring) + "\")" - else empty end, - if (.type == "Daily" or .type == "Release") - then empty else "type must be Daily or Release" end, - if (.ref_strategy == "latest_tag" or .ref_strategy == "branch_tip" or .ref_strategy == "pinned_ref") - then empty else "ref_strategy must be latest_tag, branch_tip, or pinned_ref" end, - if .type == "Daily" and (.ref_strategy != "latest_tag" and .ref_strategy != "branch_tip") - then "Daily rows must use ref_strategy=latest_tag or ref_strategy=branch_tip" - elif .type == "Release" and .ref_strategy != "pinned_ref" - then "Release rows must use ref_strategy=pinned_ref" - else empty - end, - if .ref_strategy == "latest_tag" - then required_string("tag_pattern") - elif has("tag_pattern") - then "tag_pattern is only valid with ref_strategy=latest_tag" - else empty - end, - if .type == "Release" - then required_string("target_workspace") - elif has("target_workspace") - then "target_workspace is only valid for Release" - else empty - end - ] | .[] | "row " + ($index | tostring) + " (" + (($row.kernel_variant // "unknown") | tostring) + "): " + . - end; - - if type != "object" - then "matrix root must be an object with suite_suffix_mapping and deliveries" - elif (.deliveries | type) != "array" - then "deliveries must be an array" - elif (.deliveries | length) == 0 - then "deliveries must contain at least one row" - elif (.suite_suffix_mapping | type) != "object" - then "suite_suffix_mapping is missing or not an object" - else - .deliveries as $matrix | - .suite_suffix_mapping as $mapping | - ( - [range(0; ($matrix | length)) as $index | $matrix[$index] | row_errors($index)] - + - [ - [$matrix[] | select(type == "object")] - | group_by(.kernel_variant) - | .[] - | . as $rows - | (($rows[0].kernel_variant // "unknown") | tostring) as $variant - | ([ $rows[].type ] | sort) as $types - | ([ $rows[].srcpkg ] | unique) as $srcpkgs - | ([ $rows[].binpkg ] | unique) as $binpkgs - | ([ $rows[].debian_version_stub ] | unique) as $stubs - | if ($rows | length) != 2 - then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" - elif $types != ["Daily", "Release"] - then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" - elif ($srcpkgs | length) != 1 - then "kernel_variant " + $variant + " must use one srcpkg across its Daily and Release rows" - elif ($binpkgs | length) != 1 - then "kernel_variant " + $variant + " must use one binpkg across its Daily and Release rows" - elif ($stubs | length) != 1 - then "kernel_variant " + $variant + " must use one debian_version_stub across its Daily and Release rows" - else empty - end - ] - + - [ - [ - $matrix[] - | select(type == "object") - | select((.kernel_variant | type) == "string") - | select((.srcpkg | type) == "string" and (.srcpkg | length) > 0) - | {package: .srcpkg, kernel_variant: .kernel_variant} - ] - | group_by(.package)[] - | ([.[].kernel_variant] | unique) as $variants - | select($variants | length > 1) - | "srcpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) - ] - + - [ - [ - $matrix[] - | select(type == "object") - | select((.kernel_variant | type) == "string") - | select((.binpkg | type) == "string" and (.binpkg | length) > 0) - | {package: .binpkg, kernel_variant: .kernel_variant} - ] - | group_by(.package)[] - | ([.[].kernel_variant] | unique) as $variants - | select($variants | length > 1) - | "binpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) - ] - + - [ - $mapping | to_entries[] | select(.value | type != "string") - | "suite_suffix_mapping[" + .key + "] must be a string" - ] - + - [ - $mapping - | to_entries[] - | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) - | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" - ] - + - [ - $mapping - | to_entries - | group_by(.value) - | map(select(length > 1)) - | .[]? - | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" - ] - + - [ - [$matrix[] | select(type == "object") | select((.suites | type) == "array") | .suites[]] - | unique - | .[] as $suite - | select(($mapping | has($suite)) | not) - | "suite " + $suite + " has no suite_suffix_mapping entry" - ] - ) | .[] - end -' "$MATRIX_FILE") - -if [[ -n "$validation_errors" ]]; then - echo "ERROR: Invalid kernel delivery matrix:" >&2 - while IFS= read -r error; do - [[ -n "$error" ]] && echo " - $error" >&2 - done <<< "$validation_errors" - exit 1 -fi - -result=$(jq -c \ - --arg type "$TYPE" \ - --arg single_suite "$SINGLE_SUITE" \ - --arg kernel_variant "$KERNEL_VARIANT" ' - [ - .deliveries[] - | select(.type == $type) - | select($kernel_variant == "" or .kernel_variant == $kernel_variant) - | . as $row - | ( - if $single_suite == "" - then .suites - elif (.suites | index($single_suite)) != null - then [$single_suite] - else [] - end - )[] as $suite - | $row - | del(.suites) - | . + { - "suite": $suite, - "kernel_config": ($row.kernel_config | join(",")), - "dkms": ($row.dkms | join(",")) - } - ] - | if length == 0 - then error( - "no matrix entries found for type=" + $type - + (if $kernel_variant != "" then " kernel_variant=" + $kernel_variant else "" end) - + (if $single_suite != "" then " suite=" + $single_suite else "" end) - ) - else . - end - ' "$MATRIX_FILE") || { - echo "ERROR: Matrix resolution failed for type=$TYPE${KERNEL_VARIANT:+ kernel_variant=$KERNEL_VARIANT}${SINGLE_SUITE:+ suite=$SINGLE_SUITE}" >&2 - exit 1 -} - -# Derive each leg's final debian_revision from debian_version_stub, -# suite_suffix_mapping, and its delivery type. derive-debian-revision.sh is -# the single implementation of the formula; build-kernel-deb.yml's direct -# dispatch path calls the same script for the one-suite, no-matrix case. -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -final="[]" -while IFS= read -r leg; do - suite=$(jq -r '.suite' <<< "$leg") - stub=$(jq -r '.debian_version_stub' <<< "$leg") - delivery_type=$(jq -r '.type' <<< "$leg") - variant=$(jq -r '.kernel_variant' <<< "$leg") - - revision=$("$script_dir/derive-debian-revision.sh" \ - --stub "$stub" --suite "$suite" --delivery-type "$delivery_type" \ - --matrix-file "$MATRIX_FILE") || { - echo "ERROR: Failed to derive Debian revision for kernel_variant=$variant suite=$suite type=$delivery_type" >&2 - exit 1 - } - - leg=$(jq -c --arg rev "$revision" '(. + {debian_revision: $rev}) | del(.debian_version_stub, .debian_version_suffix)' <<< "$leg") - final=$(jq -c --argjson leg "$leg" '. + [$leg]' <<< "$final") -done < <(jq -c '.[]' <<< "$result") - -echo "$final" diff --git a/debian/README.md b/debian/README.md index bdc1e2e6..0863f55c 100644 --- a/debian/README.md +++ b/debian/README.md @@ -49,6 +49,8 @@ for dependency resolution. ``` pkg-linux-qcom/ ├── build-kernel.sh ← Build orchestrator (clone → prepare → build) +├── prepare-source.sh ← Inject debian/, derive the version, run 'prepare' +├── build-source-package.sh ← Prepared tree → reproducible .orig.tar.gz, .dsc, .changes ├── debian/ │ ├── control.in ← Source-of-truth template (version-controlled) │ ├── changelog.in ← Source-of-truth template (version-controlled) @@ -184,17 +186,14 @@ KVER = + LOCALVERSION passed via `--localversion`) 3. Optionally appending `KVER_EXTRA` (passed via `--kver-extra`) -The LOCALVERSION suffix encodes the branch name and snapshot/ABI date. For -tagged builds, `build-kernel.sh` auto-extracts this from the tag name. +The LOCALVERSION suffix encodes the flavour, the snapshot date and the +commit. `prepare-source.sh` derives it from the checked-out tag or branch +with `ci/scripts/derive-localversion.sh`, the same script CI uses, so a local +build of a commit gets the version CI would give it. See +[docs/version.md](../docs/version.md) for the format. -`debian/rules` recovers LOCALVERSION from the package name at build time by -stripping the `linux-image-` prefix and the trailing `-qcom` flavour suffix: - -``` -linux-image-7.2.0-qcom-next-20260826 -→ strip "linux-image-7.0.0-rc2" → -qcom-next-20260826-qcom -→ strip "-qcom" suffix → -qcom-next-20260826 (= LOCALVERSION) -``` +`debian/rules` reads LOCALVERSION back at build time from `debian/localversion`, +which `prepare` writes, rather than recovering it from a package name. --- @@ -242,7 +241,7 @@ clone → prepare → build. Run it from the repo root. ```bash # 1. Set up kernel source -git clone git@github.com:qualcomm-linux/kernel.git kernel-source +git clone https://github.com/qualcomm-linux/kernel kernel-source cd kernel-source git checkout qcom-next @@ -274,6 +273,50 @@ python3 docker_deb_build.py \ `build-kernel.sh` calls this automatically after running `prepare`. +### Building a source package + +CI does not build binaries from a tree. It builds a Debian source package +first and hands that to the builder — Debusine for the Debian suites, sbuild +in the builder container for Ubuntu — so the thing that gets built is a +`.dsc` whose checksums are recorded in a `.changes`. The same is available +locally: + +```bash +# Clone, prepare, and stop after the source package (no kernel build) +./build-kernel.sh --latest-tag --source-package +# -> kernel-build/trixie/linux-qcom-next_.orig.tar.gz +# kernel-build/trixie/linux-qcom-next_-.dsc +# kernel-build/trixie/linux-qcom-next_-_source.changes + +# Build binaries from it, in any mode +./build-kernel.sh --dsc kernel-build/trixie/linux-qcom-next_*.dsc +./build-kernel.sh --dsc kernel-build/trixie/linux-qcom-next_*.dsc --build-mode native + +# Or step by step, from a prepared tree +./prepare-source.sh --source-dir kernel-source --distro trixie +./build-source-package.sh --source-dir kernel-source --output-dir kernel-build/trixie +``` + +The `.orig.tar.gz` is a function of the kernel commit and nothing else. It +is written with `git archive`, which gives every entry the commit's timestamp +and root ownership, and compressed with `gzip -n`, which writes no timestamp, +so two builds of one commit produce one tarball byte for byte — and so do +two suites that differ only in Debian revision, which share the tarball an +archive requires them to share. `build-source-package.sh` checks that the +commit it archives is the one the version names (`~g`), and refuses a +tree that differs from that commit outside `debian/`, since `dpkg-source` +would otherwise fold the difference into an automatic patch. An in-tree +build leaves such differences; clean them with + +```bash +git -C kernel-source checkout -- . && git -C kernel-source clean -xdf -- . ':(exclude)debian' +``` + +The debian tarball and `.dsc` are reproducible too: the changelog is dated +from the commit, and `dpkg-source` takes the tarball's timestamps from it. +Only the `gzip` implementation is outside the commit's control, which is why +CI writes every orig in one builder image. + --- ## Packages produced @@ -394,7 +437,7 @@ A listed module is a presence contract — if it fails to build, or its `BUILD_EXCLUSIVE` gates exclude this kernel, the package build fails rather than shipping a kernel image without it. To stop bundling a module, drop it from the list its build was given (for CI builds, the `dkms` field in -`ci/build-matrix.json`). +`ci/build-matrix.yaml`). ### Standalone developer use diff --git a/debian/changelog.in b/debian/changelog.in index 95b41aeb..4a67cd49 100644 --- a/debian/changelog.in +++ b/debian/changelog.in @@ -3,6 +3,7 @@ * Qualcomm ARM64 Linux kernel package * Kernel version: @KVER@ * Source: @GITCLONE@ @BRANCH@ + * Commit: @GITSHA@ * Includes kernel image, modules, DTBs, headers, and debug symbols - -- Bjordis Collaku Fri, 14 Nov 2025 10:00:00 -0800 + -- Bjordis Collaku @DATE@ diff --git a/debian/rules b/debian/rules index 1de36c97..9ab68b34 100755 --- a/debian/rules +++ b/debian/rules @@ -47,10 +47,16 @@ KREL_FILE := debian/kernel.release # debian/control before invoking any debian/rules targets. # # Inputs (make variables): -# LOCALVERSION Suffix appended to the base kernel version, e.g. -qcom-next-20260722. +# LOCALVERSION Suffix appended to the base kernel version, e.g. +qcom-next-20260722. # Required unless KVER is given explicitly. # KVER Full kernelrelease string (uname -r). Overrides LOCALVERSION. # KVER_EXTRA Optional extra suffix appended after LOCALVERSION, e.g. -ci42. +# SNAPSHOT Dated component of the Debian version: YYYYMMDD with an +# optional . ordinal, e.g. 20260722 or 20260722.1. +# Empty for branch-tip builds, which have no snapshot date. +# GITSHA Commit the build was cut from, 12 hex characters. Appended to +# the snapshot as a final discriminator. Ignored without a +# SNAPSHOT to discriminate. # DISTRO Target Debian/Ubuntu suite (default: unstable). # SRCPKG Source package name (default: linux-qcom-next). # BINPKG Binary metapackage name (default: linux-image-qcom-next). @@ -61,6 +67,20 @@ KREL_FILE := debian/kernel.release # nothing. Each name must have a -dkms package available # in the build environment; the Build-Depends entry for it is # generated from this list, so nothing else declares it. +# GIT_CLONE Kernel repository URL, recorded in debian/changelog. +# GIT_REF Resolved kernel ref (tag or branch), recorded in the changelog. +# GIT_SHA Full kernel HEAD commit SHA, recorded in the changelog. +# GITSHA above is the same commit at 12 characters, for the +# version strings; this is the unabbreviated value. +# CHANGELOG_DATE Date of the changelog entry, RFC 2822 (date -R form). +# prepare-source.sh passes the kernel commit's committer +# date, so the entry is dated by the source it describes +# and two builds of one commit write one changelog. +# dpkg-source takes the debian tarball's mtimes from this +# date and dpkg-buildpackage exports it as +# SOURCE_DATE_EPOCH, so it is also what makes the source +# package reproducible. Defaults to now, for a prepare run +# by hand. # # Outputs (generated files, all listed in debian/clean): # debian/control Substituted from debian/control.in @@ -70,18 +90,27 @@ KREL_FILE := debian/kernel.release # so an empty DKMS_MODULES is a genuine "bundle nothing" # rather than the missing-manifest misconfiguration that # the bundler rejects. -# debian/localversion Plain file containing the LOCALVERSION suffix (e.g. -qcom-next-20260722). +# debian/localversion Plain file containing the LOCALVERSION suffix (e.g. +qcom-next-20260722). # Read by override_dh_auto_build to pass LOCALVERSION= to make. # debian/pkgversion Plain file containing the full Debian package version -# (e.g. 7.2.0-rc3+20260722-0qcom1). +# (e.g. 7.2.0~rc3+git20260722-0qcom1). # Read by override_dh_gencontrol. # --------------------------------------------------------------------------- KVER_EXTRA ?= +SNAPSHOT ?= +GITSHA ?= DISTRO ?= unstable SRCPKG ?= linux-qcom-next BINPKG ?= linux-image-qcom-next DEBIAN_REVISION ?= 0qcom1 DKMS_MODULES ?= +# Source provenance recorded in debian/changelog. The version strings name the +# commit but not the repository it came from, so without these a build cannot be +# traced back to its source tree. +GIT_CLONE ?= unknown +GIT_REF ?= unknown +GIT_SHA ?= unknown +CHANGELOG_DATE ?= $(shell date -R) .PHONY: prepare prepare: @@ -93,8 +122,8 @@ prepare: else \ [ -n "$(LOCALVERSION)" ] || { \ echo "ERROR: LOCALVERSION is required when KVER is not set."; \ - echo " Pass LOCALVERSION= (e.g. LOCALVERSION=-qcom-next-20260722)"; \ - echo " or KVER= (e.g. KVER=7.2.0-rc3-qcom-next-20260722)"; \ + echo " Pass LOCALVERSION= (e.g. LOCALVERSION=+qcom-next-20260722)"; \ + echo " or KVER= (e.g. KVER=7.2.0-rc3+qcom-next-20260722)"; \ exit 1; \ }; \ echo "Reading base kernel version from kernel Makefile..."; \ @@ -112,11 +141,24 @@ prepare: echo "Appended KVER_EXTRA=$(KVER_EXTRA) -> KVER=$$KVER_RESOLVED"; \ fi; \ \ - # ── Derive package version from KVER ───────────────────────────────────── - # Package version format: +- - # where base_kver = kernel version without the LOCALVERSION suffix, - # and date = the 8-digit date component of LOCALVERSION (if present). - # For branch-tip builds (no date in LOCALVERSION), version = -. + # ── Derive package version ─────────────────────────────────────────────── + # Package version format: +git~g- + # where base_kver = kernel version without the LOCALVERSION suffix, and + # snapshot = the SNAPSHOT input (YYYYMMDD with an optional respin ordinal). + # For branch-tip builds, which have no snapshot, version = -. + # + # The snapshot orders these; the SHA only discriminates two builds that + # share one. It is joined with '~' rather than '.' because dpkg alternates + # digit and non-digit runs and reads an exhausted run as lower than a + # letter, so a '.' would put a respin BELOW the build it respins: + # + # 7.2.0+git20260826.g3f2f3ca1a81e > 7.2.0+git20260826.1.gabcdef123456 + # + # '~' sorts below everything, including the empty string, so the ordinal is + # always compared before the SHA is reached and no build has to spell a + # ".0" to hold the position. The cost is that a version reads as preceding + # the same snapshot without a SHA -- a slot nothing ever occupies, since + # every snapshot build carries one. # # Extract base_kver: read directly from kernel Makefile (same as above but # without LOCALVERSION appended), so it is always the pure kernel version. @@ -131,10 +173,23 @@ prepare: EXTRA_DEB=$$(echo "$$EXTRA" | sed 's/^-rc/~rc/'); \ BASE_KVER_CLEAN="$${VER}.$${PATCH}.$${SUB}$${EXTRA_DEB}$${TREE_LOCALVER}"; \ \ - # Extract date from LOCALVERSION: last dash-delimited component if 8 digits. - DATE_PART=$$(echo "$(LOCALVERSION)" | grep -oE '[0-9]{8}$$' || true); \ - if [ -n "$$DATE_PART" ]; then \ - UPSTREAM_VER="$${BASE_KVER_CLEAN}+$${DATE_PART}"; \ + # SNAPSHOT arrives as its own input rather than being recovered from the end + # of LOCALVERSION, because that string is not safely parseable: it carries a + # variant name whose shape is caller-supplied, and for branch-tip builds a + # 12-hex SHA that can itself end in eight digits. + if [ -n "$(SNAPSHOT)" ]; then \ + echo "$(SNAPSHOT)" | grep -qE '^[0-9]{8}(\.[0-9]+)?$$' || { \ + echo "ERROR: SNAPSHOT must be YYYYMMDD with an optional . (got '$(SNAPSHOT)')"; \ + exit 1; \ + }; \ + UPSTREAM_VER="$${BASE_KVER_CLEAN}+git$(SNAPSHOT)"; \ + if [ -n "$(GITSHA)" ]; then \ + echo "$(GITSHA)" | grep -qE '^[0-9a-f]{12}$$' || { \ + echo "ERROR: GITSHA must be 12 lowercase hex characters (got '$(GITSHA)')"; \ + exit 1; \ + }; \ + UPSTREAM_VER="$${UPSTREAM_VER}~g$(GITSHA)"; \ + fi; \ else \ UPSTREAM_VER="$${BASE_KVER_CLEAN}"; \ fi; \ @@ -193,8 +248,10 @@ prepare: -e "s|@PKGVER@|$$PKG_VERSION|g" \ -e "s|@DISTRO@|$(DISTRO)|g" \ -e "s|@KVER@|$$KVER_RESOLVED|g" \ - -e "s|@GITCLONE@||g" \ - -e "s|@BRANCH@||g" \ + -e "s|@GITCLONE@|$(GIT_CLONE)|g" \ + -e "s|@BRANCH@|$(GIT_REF)|g" \ + -e "s|@GITSHA@|$(GIT_SHA)|g" \ + -e "s|@DATE@|$(CHANGELOG_DATE)|g" \ debian/changelog.in > debian/changelog; \ \ # ── Write runtime state files ───────────────────────────────────────────── diff --git a/docs/version.md b/docs/version.md new file mode 100644 index 00000000..d27aa782 --- /dev/null +++ b/docs/version.md @@ -0,0 +1,181 @@ +# Version scheme + +This describes how a build's two version strings are composed. The examples are +`qcom-next` specific: another variant substitutes its own name and Debian +revision stub, and a variant built from a differently-shaped tag would need its +own derivation. + +A build produces two version strings, and they are deliberately not the same +string: + +```text +uname -r 7.2.0-rc7+qcom-next-20260821-gabcdef123456 +Debian version 7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 +``` + +They carry the same four facts — upstream kernel version, snapshot date, +same-day respin, commit — but they are read by two different comparators with +two different sets of rules, and each string is spelled for its own. + +## What the fields mean + +Both strings are derived from the tag and HEAD together, by +`ci/scripts/derive-localversion.sh`: + +```text +qcom-next-7.2-rc7-20260821 @ abcdef123456… + │ │ + │ └── snapshot: 20260821, optionally . + └────────── upstream kernel version: 7.2.0-rc7 +``` + +The **snapshot date** orders builds. The **respin ordinal** separates two tags +cut on the same day. The **commit** orders nothing — two SHAs have no relation — +and exists only so that a moved tag cannot produce two different kernels under +one version. It therefore comes last in both strings, after everything that does +carry ordering. + +There is no `.0` on the first tag of a day. Both comparators read an absent +ordinal as lower than a present one, so the respin already sorts above it, and +the version stays closer to the tag it came from. + +## Kernel release + +```text ++qcom-next-[.]-g<12 hex> +``` + +The full upstream version survives here, `-rc7` included: `uname -r` is the +first thing in a bug report, and it is what says whether the reporter is on a +release candidate or a stable sublevel. + +The suffix joins with `+`, not `-`. systemd compares the separator before the +chunk behind it, and `-` sorts below `+`, so `+` puts every release candidate +below the final release that follows it. Joining with `-` instead falls through +to a plain comparison of `rc` against `qcom`, where `r` > `q`, and every rc +outranks its own final release in the boot menu. This is the same trick Debian's +own kernels use (`linux-image-7.1.10+deb14-amd64`). + +The variant name is part of the string, so a flavour is a distinct kernel that +installs alongside the others rather than replacing them: + +```text +7.2.0-rc7+qcom-next-debug-20260821-gabcdef123456 +``` + +This string is also the versioned binary package name +(`linux-image-`), so a new commit means a new package name. That +is intended: it is what lets several builds coexist, and what makes the commit +recoverable from an archive listing. + +## Debian version + +```text ++git[.]~g<12 hex>- +``` + +`-rcN` becomes `~rcN`, because dpkg reads `~` as "sorts below", giving +`7.2.0~rc7 < 7.2.0`. Spelled `-rc7` it would sort *above* the release it +precedes. + +The snapshot is spelled `+git` in Debian's usual idiom for a VCS snapshot. + +The commit joins with `~`, and this is the part most likely to look like a typo. +dpkg alternates digit and non-digit runs and reads an exhausted run as lower +than a letter, so joining with `.` decides the comparison before ever reaching +the respin ordinal: + +```text +7.2.0+git20260826.g3f2f3ca1a81e > 7.2.0+git20260826.1.gabcdef123456 +``` + +— the respin sorting *below* the build it respins. `~` sorts below everything, +including the empty string, so the ordinal is always compared first and the SHA +only ever breaks a tie between builds that share a snapshot. The alternative, +spelling `.0` on every non-respin build, buys the same ordering at the cost of a +redundant ordinal in every version forever. + +The cost of `~` is one misleading reading: the version sorts below the same +snapshot without a SHA, as though it preceded it. Nothing occupies that slot, +because every snapshot build carries a SHA. + +The revision (`0qli1~bpo13+1`) is not derived. Each matrix entry states its own +outright, and it says two things: where the suite belongs relative to the other +suites, and which packaging built it. The `~bpo13+1` is the backports +convention, sorting a trixie build below a forky build of the same kernel; the +trailing digit on the `0qli` stub is the packaging revision, bumped when the +packaging changes but the kernel snapshot does not. See the matrix +documentation in the top-level [README](../README.md#matrix-model). + +Nothing in the revision marks how far a build has got. A kernel is built once +and the artifact that build produced is what any archive holds, so there is no +second version for a marker to sort against. + +## Ordering + +The full chain for one suite, in the order dpkg sorts it: + +```text +7.2.0~rc7+git20260820.1~g011a82096bee-0qli1~bpo13+1 first tag of the 20th +7.2.0~rc7+git20260820.2~g3f2f3ca1a81e-0qli1~bpo13+1 respin, same day +7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 next snapshot +7.2.0~rc7+git20260821~gabcdef123456-0qli2~bpo13+1 packaging rebuild +7.2.0+git20260902~g123456789abc-0qli1~bpo13+1 7.2 final +``` + +## Moved tags + +If an upstream tag is re-cut against a different commit, the SHA changes and so +does the version, so the two builds cannot be mistaken for each other. But +nothing guarantees the replacement sorts *above* the original — two hashes have +no order — so apt may not offer it as an upgrade: + +```text +7.2.0~rc7+git20260821~g011a82096bee-0qli1~bpo13+1 original +7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 retagged; happens to sort above +7.2.0~rc7+git20260821~g0009f3c1d2e4-0qli1~bpo13+1 retagged; sorts BELOW, no upgrade +``` + +Whether it lands above or below is down to the hex, so treat a moved tag as +needing a version bump of its own — the respin ordinal, or the packaging +revision: + +```text +7.2.0~rc7+git20260821.1~g0009f3c1d2e4-0qli1~bpo13+1 supersedes it either way +``` + +## Branch-tip builds + +A build from a branch rather than a tag has no tag date, so the date comes from +the HEAD commit instead. The result has the same shape as a tag build and orders +in the same sequence: + +```text +uname -r 7.2.0-rc7+qcom-next-20260904-g07f50dc44edd +Debian version 7.2.0~rc7+git20260904~g07f50dc44edd-0qli1~bpo13+1 +``` + +It is the *committer* date of the commit, not the time the build ran. That means +rebuilding a commit reproduces its version instead of inventing a higher one, +and the date describes the source rather than when CI happened to start. Author +dates are not used, because a backported patch can carry one months old. + +The catch is that a build clock only ever moves forwards, and a commit date does +not. If the branch is rewound to an older commit, the next build's version goes +*down*, and apt will not offer it as an upgrade. That is arguably the honest +answer — older source, older version — but it is the one case where dating by +the clock would behave differently. + +## What is not in the version + +The version strings name the commit, but not the tree it came from. The +repository, the resolved ref and the full 40-character SHA are recorded in the +package changelog instead: + +```text + * Kernel version: 7.2.0-rc7+qcom-next-20260904-g07f50dc44edd + * Source: https://github.com/qualcomm-linux/kernel qcom-next + * Commit: 07f50dc44edd… +``` + +so `apt changelog` on an installed image is enough to find the exact source. diff --git a/prepare-source.sh b/prepare-source.sh index f0cbbc72..c33b0551 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -20,6 +20,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEFAULT_DISTRO="trixie" +DEFAULT_FLAVOUR="qcom-next" DEFAULT_SRCPKG="linux-qcom-next" DEFAULT_BINPKG="linux-image-qcom-next" DEFAULT_DEBIAN_REVISION="0qcom1" @@ -51,11 +52,34 @@ OPTIONS: Version control: -d, --distro DISTRO Target suite: trixie|forky|sid|noble|questing|resolute (default: $DEFAULT_DISTRO) + --flavour NAME Kernel flavour, the identity LOCALVERSION carries + (default: $DEFAULT_FLAVOUR). Only consulted when + --localversion is not given. --localversion SUFFIX LOCALVERSION suffix appended to the base kernel - version (e.g. -qcom-next-20260722). - Auto-detected from git tag if not specified. + version (e.g. +qcom-next-20260722-g07f50dc44edd). + Derived from the checked-out tag or branch by + ci/scripts/derive-localversion.sh if not given. + --snapshot SNAPSHOT Dated component of the Debian version: YYYYMMDD + with an optional . ordinal (e.g. + 20260722 or 20260722.1). Derived alongside + --localversion; pass it explicitly whenever + --localversion is passed explicitly. + --git-sha SHA Full commit SHA the build was cut from. Its + first 12 characters discriminate two builds of + one snapshot (a moved tag) in the version + strings; the full value is recorded in the + changelog. Auto-detected from HEAD. --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). + --git-clone URL Kernel repository URL, recorded in the changelog. + Defaults to the checkout's origin remote. + --git-ref REF Resolved kernel ref (tag or branch), recorded in + the changelog. Defaults to the exact tag HEAD + sits on, or the branch it is the tip of. + --changelog-date DATE Date of the changelog entry, RFC 2822 (date -R + form). Defaults to HEAD's committer date, so + the entry is dated by the source and the + source package is reproducible. Package naming: --srcpkg NAME Source package name (default: $DEFAULT_SRCPKG) @@ -95,13 +119,14 @@ OPTIONS: -h, --help Show this help EXAMPLES: - # Minimal: auto-detect LOCALVERSION from git tag, default package names + # Minimal: derive LOCALVERSION from the checkout, default package names $0 --source-dir /path/to/kernel # Full CI invocation with all options $0 --source-dir /path/to/kernel \\ --distro trixie \\ - --localversion -qcom-next-20260722 \\ + --localversion +qcom-next-20260722-g07f50dc44edd \\ + --snapshot 20260722 \\ --srcpkg linux-qcom-next \\ --binpkg linux-image-qcom-next \\ --debian-revision 0qcom1 \\ @@ -114,20 +139,32 @@ EOF # Defaults SOURCE_DIR="" DISTRO="$DEFAULT_DISTRO" +FLAVOUR="$DEFAULT_FLAVOUR" LOCALVERSION="" +SNAPSHOT="" KVER_EXTRA="" SRCPKG="$DEFAULT_SRCPKG" BINPKG="$DEFAULT_BINPKG" DEBIAN_REVISION="$DEFAULT_DEBIAN_REVISION" KERNEL_CONFIG="" DKMS_MODULES="" +GIT_CLONE="" +GIT_REF="" +GIT_SHA="" +CHANGELOG_DATE="" while [[ $# -gt 0 ]]; do case $1 in -s|--source-dir) SOURCE_DIR="$2"; shift 2 ;; -d|--distro) DISTRO="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; + --snapshot) SNAPSHOT="$2"; shift 2 ;; + --git-sha) GIT_SHA="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; + --git-clone) GIT_CLONE="$2"; shift 2 ;; + --git-ref) GIT_REF="$2"; shift 2 ;; + --changelog-date) CHANGELOG_DATE="$2"; shift 2 ;; --srcpkg) SRCPKG="$2"; shift 2 ;; --binpkg) BINPKG="$2"; shift 2 ;; --debian-revision) DEBIAN_REVISION="$2"; shift 2 ;; @@ -152,37 +189,83 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -d "$DEBIAN_DIR" ]] || { log_error "Debian dir not found: $DEBIAN_DIR"; exit 1; } -# ── Helper: derive LOCALVERSION from a tag name ────────────────────────────── -# qcom-next-7.2-rc3-20260722 -> -qcom-next-20260722 -_auto_localversion() { - local tag="$1" - if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$ ]]; then - echo "-${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" - else - echo "-$tag" - fi -} - -# ── Auto-detect LOCALVERSION from git tag (if not provided) ────────────────── +# ── Resolve the commit, once ───────────────────────────────────────────────── +# One SHA, used at two widths: the first 12 characters go in the version strings +# (short enough to keep a boot menu readable), the full value goes in the +# changelog. Deriving one from the other is what keeps them the same commit. +[[ -n "$GIT_SHA" ]] || GIT_SHA=$(git -C "$SOURCE_DIR" rev-parse HEAD 2>/dev/null || true) +GITSHA="${GIT_SHA:0:12}" + +# ── Name the checkout ──────────────────────────────────────────────────────── +# The ref HEAD answers to: the exact tag it sits on, else the branch it is the +# tip of. A detached HEAD reports HEAD, which is treated like any undated ref. +# This names the checkout for the version derivation below and, when the +# caller gave none, for the provenance recorded in the changelog -- so a local +# build says where it came from just as a CI build does. +# +# Every git call here tolerates failure. CI passes all of these values in and +# runs this script in a container as a different user from the one owning +# the checkout, where git refuses to read the repository at all; nothing it +# was given is then asked of git. +_git() { git -C "$SOURCE_DIR" "$@" 2>/dev/null || true; } +CHECKOUT_REF="" +if [[ -z "$GIT_REF" || -z "$LOCALVERSION" ]]; then + CHECKOUT_REF=$(_git describe --tags --exact-match) + [[ -n "$CHECKOUT_REF" ]] || CHECKOUT_REF=$(_git rev-parse --abbrev-ref HEAD) +fi +[[ -n "$GIT_CLONE" ]] || GIT_CLONE=$(_git remote get-url origin) +[[ -n "$GIT_REF" ]] || GIT_REF="$CHECKOUT_REF" +# Committer date, like the snapshot: it describes the source rather than the +# build, so rebuilding a commit rewrites the same changelog entry. +[[ -n "$CHANGELOG_DATE" ]] || CHANGELOG_DATE=$(_git log -1 --format=%cD) + +# ── Derive LOCALVERSION, SNAPSHOT and GITSHA from the checkout (if not given) ── +# The same derivation CI performs, by the same script: the ref is the exact +# tag HEAD sits on, or the branch it is the tip of, and the date is HEAD's +# committer date for the branch-tip case. A local build of a commit therefore +# produces the version CI would give it, which is what lets a source package +# built here stand in for one built there. if [[ -z "$LOCALVERSION" ]]; then - GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) - if [[ -n "$GIT_TAG" ]]; then - LOCALVERSION="$(_auto_localversion "$GIT_TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' from tag '$GIT_TAG'" + if [[ -z "$GIT_SHA" || -z "$CHECKOUT_REF" ]]; then + log_warn "LOCALVERSION not set and $SOURCE_DIR is not a readable git checkout." + log_warn "Package will be named linux-image- (no flavour/date suffix)." + log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722-g07f50dc44edd" else - log_warn "LOCALVERSION not set and no exact git tag found." - log_warn "Package will be named linux-image- (no branch/date suffix)." - log_warn "Use --localversion to specify, e.g.: --localversion -qcom-next-20260722" + # Committer date, normalised to UTC, as CI does. + DERIVE_DATE=$(TZ=UTC git -C "$SOURCE_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) + FIELDS=$("$SCRIPT_DIR/ci/scripts/derive-localversion.sh" \ + --flavour "$FLAVOUR" \ + --ref "$CHECKOUT_REF" \ + --sha "$GIT_SHA" \ + --date "$DERIVE_DATE") + while IFS='=' read -r key value; do + case "$key" in + LOCALVERSION) LOCALVERSION="$value" ;; + SNAPSHOT) SNAPSHOT="$value" ;; + GITSHA) GITSHA="$value" ;; + *) log_error "Unexpected field '$key' from derive-localversion.sh"; exit 1 ;; + esac + done <<< "$FIELDS" + log_info "Derived LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from $CHECKOUT_REF" fi +elif [[ -z "$SNAPSHOT" ]]; then + # An explicit --localversion is not parsed for a snapshot; say so rather + # than silently dropping the dated component from the Debian version. + log_warn "--localversion given without --snapshot: the Debian version will" + log_warn "carry no +git component. Pass --snapshot to supply one." fi log_step "Configuration:" log_info " Source dir: $SOURCE_DIR" log_info " Distro: $DISTRO" +log_info " Flavour: $FLAVOUR" log_info " Source package: $SRCPKG" log_info " Binary metapkg: $BINPKG" log_info " Debian revision: $DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" +[[ -n "$SNAPSHOT" ]] && log_info " SNAPSHOT: $SNAPSHOT" +[[ -n "$GITSHA" ]] && log_info " GITSHA: $GITSHA" +[[ -n "$GIT_CLONE" ]] && log_info " Source: $GIT_CLONE $GIT_REF" [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" [[ -n "$KERNEL_CONFIG" ]] && log_info " Kernel config: $KERNEL_CONFIG" [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" @@ -296,12 +379,25 @@ fi log_step "Running debian/rules prepare..." PREPARE_ARGS="DISTRO=$DISTRO SRCPKG=$SRCPKG BINPKG=$BINPKG DEBIAN_REVISION=$DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && PREPARE_ARGS="$PREPARE_ARGS LOCALVERSION=$LOCALVERSION" +[[ -n "$SNAPSHOT" ]] && PREPARE_ARGS="$PREPARE_ARGS SNAPSHOT=$SNAPSHOT" +[[ -n "$GITSHA" ]] && PREPARE_ARGS="$PREPARE_ARGS GITSHA=$GITSHA" [[ -n "$KVER_EXTRA" ]] && PREPARE_ARGS="$PREPARE_ARGS KVER_EXTRA=$KVER_EXTRA" +# Source provenance for debian/changelog. LOCALVERSION identifies a build by +# date, not by commit, so the SHA recorded here is what makes a build traceable +# back to exact source -- particularly for branch-tip builds. +[[ -n "$GIT_CLONE" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_CLONE=$GIT_CLONE" +[[ -n "$GIT_REF" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_REF=$GIT_REF" +[[ -n "$GIT_SHA" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_SHA=$GIT_SHA" +# Quoted separately: an RFC 2822 date has spaces, which the string-built +# argument list above would split. Empty means "now", which debian/rules +# decides for itself. +CHANGELOG_DATE_ARG=() +[[ -n "$CHANGELOG_DATE" ]] && CHANGELOG_DATE_ARG=("CHANGELOG_DATE=$CHANGELOG_DATE") # Spaces are stripped so a list written as "kgsl, camx" stays a single make # argument; debian/rules validates the names it is given. [[ -n "$DKMS_MODULES" ]] && PREPARE_ARGS="$PREPARE_ARGS DKMS_MODULES=$(tr -d ' ' <<< "$DKMS_MODULES")" # shellcheck disable=SC2086 -make -f "$SOURCE_DIR/debian/rules" -C "$SOURCE_DIR" prepare $PREPARE_ARGS +make -f "$SOURCE_DIR/debian/rules" -C "$SOURCE_DIR" prepare $PREPARE_ARGS "${CHANGELOG_DATE_ARG[@]}" echo log_step "Source preparation complete."