From e82486abb9251a3bf60f1c07ff6e321e13cda8c7 Mon Sep 17 00:00:00 2001 From: umair Date: Thu, 3 Sep 2026 14:01:57 +0100 Subject: [PATCH] Add lockstep release workflow to main (dispatch-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings `.github/workflows/release.yml` and `scripts/release_preflight.py` from #684 (`pubsub-split/release-tooling`) onto `main`, and nothing else. GitHub only registers a `workflow_dispatch` workflow once its file exists on the default branch. The PubSub split work merges to `integration/v4` and will not reach `main` until GA, so without this the lockstep release workflow cannot be dispatched at all — including the run that claims the `ably-pubsub-core` / `ably-pubsub-server` names on PyPI through trusted publishing (plan steps 1 and 16) and the prerelease runs cut from split branches. With the file here, `gh workflow run release.yml --ref -f version=...` works against any ref that carries the split layout. The workflow is inert on `main`: dispatched here, the pre-flight refuses the single-distribution layout before anything is built or uploaded, and publishing would in any case need trusted publishers bound to the two new project names, which do not exist yet. To make that refusal legible rather than a FileNotFoundError, the pre-flight grows a `require_split_layout()` guard that aborts with an explanation when `core/pyproject.toml` / `server/pyproject.toml` are absent, and `release.yml` runs the (build-free) version-site pre-flight right after `uv sync`, before the build, so the guard fires first. Both are no-ops on the split layout; #684 should pick up the same two changes so the copies stay identical. Mirrors ably-ruby#455. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release.yml | 208 ++++++++++++++++------- scripts/release_preflight.py | 306 ++++++++++++++++++++++++++++++++++ 2 files changed, 452 insertions(+), 62 deletions(-) create mode 100644 scripts/release_preflight.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f47e6b0..1139a389 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,57 @@ -name: Publish Python distribution to PyPI +# Releases ably-pubsub-core and ably-pubsub-server to PyPI in lockstep. +# +# LOCKSTEP. This repository builds two distributions and they are always +# released together at one version: ably-pubsub-server pins +# `ably-pubsub-core==` exactly, so a server release without its +# core is uninstallable and a core release without its server is invisible. +# +# PRE-FLIGHT. Everything that can be checked is checked before the first +# upload, by scripts/release_preflight.py — the same script check.yml's +# release-dry-run job runs on every pull request. It verifies the release +# version against every version site (both pyproject `version` fields, the +# core's `lib_version`, the server's `__version__`) and the server's exact core +# pins, that dist/ holds exactly one wheel and one sdist per distribution, that +# the core artifacts carry the generated `ably_pubsub/core/sync/` flavour, that +# neither wheel ships `ably_pubsub/__init__.py` (the namespace must stay PEP +# 420) or any file the other wheel also ships, and that `twine check` passes. +# +# CORE BEFORE SERVER. The two publish steps are ordered, so the server is never +# visible on the index before the core version it pins. +# +# PARTIAL RELEASES ARE RE-RUNNABLE, NOT IMPOSSIBLE. PyPI has no cross-project +# transaction: two projects means two uploads, and the second can fail after +# the first succeeded. Both steps therefore set `skip-existing: true`, so +# re-running this workflow at the same version skips whatever already landed +# and completes the release. Never bump the version to work around a partial +# release — re-run it. +# +# TRUSTED PUBLISHING. Both projects must have a trusted publisher configured on +# pypi.org (and on test.pypi.org) bound to this repository, this workflow file +# (`release.yml`) and the `pypi` / `testpypi` environment respectively — plan +# step 16, done immediately after the repo rename so the binding is made once +# against the new name. PyPI's OIDC token covers every project that trusts the +# requesting configuration, so one job's `id-token: write` publishes both. +# The `pypi` environment's required-reviewer rule is the human approval gate. +# +# TRIGGERS. A `v` tag push releases to PyPI. A manual dispatch always +# goes to TestPyPI and only reaches PyPI when `publish` is set — which is how +# prereleases are cut from a branch (`gh workflow run release.yml --ref +# -f version=4.0.0rc1 -f publish=true`). `workflow_dispatch` only +# works for workflow files present on the default branch. +name: Publish Python distributions to PyPI on: workflow_dispatch: + inputs: + version: + description: 'Version to release, e.g. 4.0.0 or 4.0.0rc1 — must equal every version site and the server''s core pin' + required: true + type: string + publish: + description: 'Also publish to PyPI (not just TestPyPI)' + required: false + default: false + type: boolean push: tags: - 'v[0-9]+.[0-9]+.[0-9]+*' @@ -10,16 +60,33 @@ permissions: {} jobs: build: - name: Build distribution 📦 + name: Build distributions 📦 runs-on: ubuntu-latest permissions: contents: read + outputs: + version: ${{ steps.release-version.outputs.version }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: submodules: 'recursive' persist-credentials: false + + - name: Determine the release version + id: release-version + env: + VERSION_INPUT: ${{ inputs.version }} + run: | + set -euo pipefail + if [ -n "${VERSION_INPUT}" ]; then + RELEASE_VERSION="${VERSION_INPUT}" + else + RELEASE_VERSION="${GITHUB_REF#refs/tags/v}" + fi + echo "Releasing ${RELEASE_VERSION}" + echo "version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + - name: Set up Python 3.12 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 id: setup-python @@ -32,49 +99,49 @@ jobs: enable-cache: false - name: Install dependencies - run: uv sync --extra crypto --extra dev - - name: Generate rest sync code and tests + run: uv sync + + # Cheap checks that need no build: the version sites, and — first of all + # — that this ref actually has the split layout. This workflow file also + # lives on `main` (workflow_dispatch only offers workflows present on the + # default branch), where it is inert: dispatched there, the pre-flight + # stops here with an explanation instead of a confusing build error. + - name: 'Pre-flight: layout and version sites (before anything is built)' + env: + RELEASE_VERSION: ${{ steps.release-version.outputs.version }} + run: uv run python scripts/release_preflight.py --version "$RELEASE_VERSION" + + - name: Generate the sync flavour run: uv run unasync - - name: Build a binary wheel and a source tarball - run: uv build + - name: Build both distributions into one dist/ + run: | + set -euo pipefail + uv build --package ably-pubsub-core --out-dir dist + uv build --package ably-pubsub-server --out-dir dist + + # Nothing below this point is reversible, so this is the last chance to + # refuse the release. Same script as check.yml's release-dry-run job. + - name: 'Pre-flight: nothing is uploaded unless everything agrees' + env: + RELEASE_VERSION: ${{ steps.release-version.outputs.version }} + run: uv run python scripts/release_preflight.py --version "$RELEASE_VERSION" dist/ + - name: Store the distribution packages uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: python-package-distributions path: dist/ - - name: Check that wheel and tarball contains ably/sync/ - run: | - # Check wheel - WHEEL=$(ls dist/*.whl | head -n 1) - echo "Checking wheel: $WHEEL" - if unzip -l "$WHEEL" | grep -q "ably/sync/"; then - echo "✅ Found ably/sync/ in wheel" - else - unzip -l "$WHEEL" - echo "❌ ably/sync/ not found in wheel" - exit 1 - fi - - # Check tarball - TARBALL=$(ls dist/*.tar.gz | head -n 1) - echo "Checking tarball: $TARBALL" - if tar -tzf "$TARBALL" | grep -q "ably/sync/"; then - echo "✅ Found ably/sync/ in tarball" - else - tar -tzf "$TARBALL" - echo "❌ ably/sync/ not found in tarball" - exit 1 - fi - publish-to-pypi: - name: Publish Python distribution to PyPI - if: startsWith(github.ref, 'refs/tags/v') # only publish to PyPI on tag pushes + publish-to-testpypi: + name: Publish distributions 📦 to TestPyPI needs: - build runs-on: ubuntu-latest + environment: - name: pypi - url: https://pypi.org/p/ably + name: testpypi + url: https://test.pypi.org/p/ably-pubsub-server + permissions: id-token: write # IMPORTANT: mandatory for trusted publishing @@ -85,41 +152,41 @@ jobs: name: python-package-distributions path: dist/ - - name: Extract tag - id: tag + # pypa/gh-action-pypi-publish uploads a whole directory, so the two + # projects are split into two directories to be uploaded in order. + - name: Split the dists by project run: | - TAG=${GITHUB_REF#refs/tags/v} - echo "tag=$TAG" >> $GITHUB_OUTPUT + set -euo pipefail + mkdir -p dist-core dist-server + mv dist/ably_pubsub_core-* dist-core/ + mv dist/ably_pubsub_server-* dist-server/ - - name: Read VERSION_NAME from dist/ - id: version - run: | - VERSION_NAME=$(basename dist/ably-*.tar.gz | sed -E 's/^ably-([^-]+)\.tar\.gz$/\1/') - echo "version=$VERSION_NAME" >> $GITHUB_OUTPUT - - - name: Compare version with tag - run: | - if [ "$VERSION" != "$TAG" ]; then - echo "VERSION ($VERSION) does not match tag ($TAG)." - exit 1 - fi - env: - VERSION: ${{ steps.version.outputs.version }} - TAG: ${{ steps.tag.outputs.tag }} + - name: Publish ably-pubsub-core 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + with: + packages-dir: dist-core/ + repository-url: https://test.pypi.org/legacy/ + skip-existing: true - - name: Publish distribution 📦 to PyPI + - name: Publish ably-pubsub-server 📦 to TestPyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + with: + packages-dir: dist-server/ + repository-url: https://test.pypi.org/legacy/ + skip-existing: true - publish-to-testpypi: - name: Publish Python distribution to TestPyPI + publish-to-pypi: + name: Publish distributions 📦 to PyPI + # Tag pushes release; a manual dispatch has to opt in explicitly. + if: startsWith(github.ref, 'refs/tags/v') || inputs.publish + # Deliberately not `needs: publish-to-testpypi`: TestPyPI is a staging + # signal, not a gate — an outage there must not block a real release. needs: - build runs-on: ubuntu-latest - environment: - name: testpypi - url: https://test.pypi.org/p/ably - + name: pypi + url: https://pypi.org/p/ably-pubsub-server permissions: id-token: write # IMPORTANT: mandatory for trusted publishing @@ -129,7 +196,24 @@ jobs: with: name: python-package-distributions path: dist/ - - name: Publish distribution 📦 to TestPyPI + + - name: Split the dists by project + run: | + set -euo pipefail + mkdir -p dist-core dist-server + mv dist/ably_pubsub_core-* dist-core/ + mv dist/ably_pubsub_server-* dist-server/ + + # Core first: the server pins this exact version, so it must never be the + # one that is visible on the index alone. + - name: Publish ably-pubsub-core 📦 to PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: - repository-url: https://test.pypi.org/legacy/ + packages-dir: dist-core/ + skip-existing: true + + - name: Publish ably-pubsub-server 📦 to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + with: + packages-dir: dist-server/ + skip-existing: true diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py new file mode 100644 index 00000000..f8723862 --- /dev/null +++ b/scripts/release_preflight.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Pre-flight checks for a lockstep release of ably-pubsub-core and ably-pubsub-server. + +Both distributions are released at one version and the server pins the core +exactly, so a release is only meaningful if every version site agrees and both +sets of artifacts are well formed. PyPI has no cross-project transaction: once +the core is uploaded it cannot be taken back, so everything that can be checked +must be checked *before* the first upload. + +This is the single implementation behind both callers: + + # release.yml — the version comes from the tag or the dispatch input + uv run python scripts/release_preflight.py --version 4.0.0 dist/ + + # check.yml release-dry-run — no authoritative version, just internal agreement + uv run python scripts/release_preflight.py dist/ + + # /release skill and local use — version sites only, no build needed + uv run python scripts/release_preflight.py + +Exit status is 0 when every check passes and 1 otherwise, with every failure +reported (the run does not stop at the first one). +""" + +import argparse +import re +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +CORE_PYPROJECT = REPO_ROOT / 'core' / 'pyproject.toml' +SERVER_PYPROJECT = REPO_ROOT / 'server' / 'pyproject.toml' +CORE_INIT = REPO_ROOT / 'core' / 'src' / 'ably_pubsub' / 'core' / '__init__.py' +SERVER_INIT = REPO_ROOT / 'server' / 'src' / 'ably_pubsub' / 'server' / '__init__.py' + +CORE_DIST = 'ably_pubsub_core' +SERVER_DIST = 'ably_pubsub_server' + +# PEP 440, the subset a release can actually be: a release segment, an optional +# pre/post/dev suffix. Local versions are excluded — PyPI rejects them. +PEP_440 = re.compile( + r'^([1-9][0-9]*!)?' + r'(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*' + r'((a|b|rc)(0|[1-9][0-9]*))?' + r'(\.post(0|[1-9][0-9]*))?' + r'(\.dev(0|[1-9][0-9]*))?$' +) + + +def require_split_layout(): + """Refuse to run against a tree that does not have the split layout. + + This script (and the release workflow that calls it) also exists on `main`, + because `workflow_dispatch` only offers a workflow that is present on the + default branch — see the plan's step 15b. `main` still has the single flat + `ably/` package, so a dispatch there must stop here with an explanation + rather than a FileNotFoundError from the first version site it reads. + + On the split layout every path below exists, so this is a no-op. + """ + missing = [ + str(path.relative_to(REPO_ROOT)) + for path in (CORE_PYPROJECT, SERVER_PYPROJECT) + if not path.is_file() + ] + if missing: + raise SystemExit( + 'pre-flight: this workflow releases the split distributions ' + '(ably-pubsub-core and ably-pubsub-server), but this ref still has the ' + 'single `ably` layout — no ' + ' or '.join(missing) + '. There is nothing ' + 'here to release in lockstep. Dispatch this workflow against a ref that ' + 'has the split layout (--ref integration/v4, or a pubsub-split/* branch); ' + 'releases of the legacy `ably` distribution are cut from its maintenance ' + 'branch with that branch\'s own single-distribution release.yml.' + ) + + +class Failures: + def __init__(self): + self.messages = [] + + def add(self, message): + self.messages.append(message) + + def check(self, condition, message): + if not condition: + self.add(message) + return bool(condition) + + +def read_scalar(path, pattern, what): + """Read a single quoted scalar out of a file. + + Deliberately regex rather than tomllib/import: tomllib only arrived in 3.11, + and importing the packages would make the pre-flight depend on them being + installed rather than on what is actually in the tree about to be released. + """ + match = re.search(pattern, path.read_text(), re.MULTILINE) + if not match: + raise SystemExit(f'pre-flight could not read {what} from {path}') + return match.group(1) + + +def version_sites(): + return { + f'{CORE_PYPROJECT.relative_to(REPO_ROOT)} [project] version': + read_scalar(CORE_PYPROJECT, r'^version = "([^"]+)"', 'version'), + f'{SERVER_PYPROJECT.relative_to(REPO_ROOT)} [project] version': + read_scalar(SERVER_PYPROJECT, r'^version = "([^"]+)"', 'version'), + f'{CORE_INIT.relative_to(REPO_ROOT)} lib_version': + read_scalar(CORE_INIT, r"^lib_version = '([^']+)'", 'lib_version'), + f'{SERVER_INIT.relative_to(REPO_ROOT)} __version__': + read_scalar(SERVER_INIT, r"^__version__ = '([^']+)'", '__version__'), + } + + +def core_pins(): + """Every `ably-pubsub-core[...]==` pin in the server's pyproject.""" + text = SERVER_PYPROJECT.read_text() + return set(re.findall(r'"ably-pubsub-core(?:\[[\w,]+\])?==([^"]+)"', text)) + + +def core_extras(): + """The names in the core's `[project.optional-dependencies]` table.""" + text = CORE_PYPROJECT.read_text() + section = text.split('[project.optional-dependencies]', 1)[1] + section = re.split(r'^\[', section, maxsplit=1, flags=re.MULTILINE)[0] + return re.findall(r'^(\w+) = \[', section, re.MULTILINE) + + +def check_versions(failures, expected): + sites = version_sites() + pins = core_pins() + + if expected is None: + # No authoritative version: the sites only have to agree with each other, + # and the first one is then taken as the release version for the rest. + expected = sorted(sites.values())[0] + + failures.check( + PEP_440.match(expected), + f'version {expected!r} is not a valid PEP 440 public version', + ) + + for site, value in sorted(sites.items()): + failures.check(value == expected, f'{site} is {value!r}, expected {expected!r}') + + failures.check(pins, 'no ably-pubsub-core== pin found in server/pyproject.toml') + for pin in sorted(pins): + failures.check( + pin == expected, + f'server/pyproject.toml pins ably-pubsub-core=={pin}, expected =={expected}', + ) + + # Every extra the core offers must be forwarded, or `pip install + # ably-pubsub-server[crypto]` silently installs a core without the extra. + server_text = SERVER_PYPROJECT.read_text() + for extra in core_extras(): + failures.check( + f'"ably-pubsub-core[{extra}]=={expected}"' in server_text, + f'server/pyproject.toml does not forward the core extra {extra!r} at {expected}', + ) + + return expected + + +def one_of_each(failures, dist_dir, version): + """Exactly one wheel and one sdist per distribution, at the release version.""" + found = {} + for name in (CORE_DIST, SERVER_DIST): + wheels = sorted(dist_dir.glob(f'{name}-*.whl')) + sdists = sorted(dist_dir.glob(f'{name}-*.tar.gz')) + failures.check(len(wheels) == 1, f'expected exactly one {name} wheel in {dist_dir}, found {len(wheels)}') + failures.check(len(sdists) == 1, f'expected exactly one {name} sdist in {dist_dir}, found {len(sdists)}') + for path in wheels + sdists: + failures.check( + f'-{version}' in path.name.replace(f'{name}', '', 1), + f'{path.name} is not version {version}', + ) + found[name] = (wheels[0] if wheels else None, sdists[0] if sdists else None) + + unexpected = sorted( + p.name for p in dist_dir.iterdir() + if p.suffix in {'.whl', '.gz'} and not p.name.startswith((CORE_DIST + '-', SERVER_DIST + '-')) + ) + failures.check(not unexpected, f'unexpected files in {dist_dir}: {", ".join(unexpected)}') + return found + + +def wheel_names(path): + with zipfile.ZipFile(path) as zf: + return set(zf.namelist()) + + +def sdist_names(path): + with tarfile.open(path) as tf: + # Strip the `-/` prefix so paths compare like the wheel's. + return {n.split('/', 1)[1] for n in tf.getnames() if '/' in n} + + +def check_artifacts(failures, dist_dir, version): + found = one_of_each(failures, dist_dir, version) + + core_wheel, core_sdist = found[CORE_DIST] + server_wheel, server_sdist = found[SERVER_DIST] + + # The generated sync flavour is not in git; a core built without running + # unasync first looks fine until someone imports ably_pubsub.core.sync. + if core_wheel: + names = wheel_names(core_wheel) + failures.check( + any(n.startswith('ably_pubsub/core/sync/') for n in names), + f'{core_wheel.name} does not contain ably_pubsub/core/sync/ (was unasync run?)', + ) + if core_sdist: + names = sdist_names(core_sdist) + failures.check( + any('ably_pubsub/core/sync/' in n for n in names), + f'{core_sdist.name} does not contain ably_pubsub/core/sync/ (was unasync run?)', + ) + + # PEP 420: an __init__.py here would belong to whichever distribution shipped + # it, so removing that one would delete the other's entry into the namespace. + for wheel in (core_wheel, server_wheel): + if wheel: + failures.check( + 'ably_pubsub/__init__.py' not in wheel_names(wheel), + f'{wheel.name} ships ably_pubsub/__init__.py; the namespace must stay PEP 420', + ) + + # Two wheels writing the same path into site-packages means installing one + # overwrites the other and uninstalling either breaks what is left. + if core_wheel and server_wheel: + core_files = {n for n in wheel_names(core_wheel) if not n.endswith('/')} + server_files = {n for n in wheel_names(server_wheel) if not n.endswith('/')} + overlap = sorted(core_files & server_files) + failures.check(not overlap, f'the two wheels both ship: {", ".join(overlap)}') + + # Both distributions must carry the licence they claim in their metadata. + for path in (core_wheel, server_wheel): + if path: + failures.check( + any(n.endswith('.dist-info/licenses/LICENSE') for n in wheel_names(path)), + f'{path.name} does not bundle LICENSE', + ) + for path in (core_sdist, server_sdist): + if path: + failures.check('LICENSE' in sdist_names(path), f'{path.name} does not bundle LICENSE') + + artifacts = sorted( + str(p) for p in dist_dir.iterdir() + if p.is_file() and (p.name.endswith('.whl') or p.name.endswith('.tar.gz')) + ) + result = subprocess.run( + [sys.executable, '-m', 'twine', 'check', *artifacts], + capture_output=True, + text=True, + ) + sys.stdout.write(result.stdout) + sys.stdout.write(result.stderr) + failures.check(result.returncode == 0, 'twine check failed') + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + '--version', + help='the version being released (from the tag or the workflow_dispatch input). ' + 'Without it the version sites only have to agree with each other.', + ) + parser.add_argument( + 'dist', + nargs='?', + type=Path, + help='directory holding the built distributions. Omit to check the version sites only.', + ) + args = parser.parse_args(argv) + + require_split_layout() + + failures = Failures() + version = check_versions(failures, args.version) + + if args.dist is not None: + if not args.dist.is_dir(): + raise SystemExit(f'pre-flight: {args.dist} is not a directory') + check_artifacts(failures, args.dist, version) + + if failures.messages: + print(f'\nPre-flight FAILED ({len(failures.messages)} problem(s)); nothing should be published:\n', + file=sys.stderr) + for message in failures.messages: + print(f' - {message}', file=sys.stderr) + return 1 + + scope = 'version sites' if args.dist is None else f'version sites and the artifacts in {args.dist}' + print(f'Pre-flight OK: {scope} all agree on {version}.') + return 0 + + +if __name__ == '__main__': + sys.exit(main())