diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f47e6b0..1139a389 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,57 @@ -name: Publish Python distribution to PyPI +# Releases ably-pubsub-core and ably-pubsub-server to PyPI in lockstep. +# +# LOCKSTEP. This repository builds two distributions and they are always +# released together at one version: ably-pubsub-server pins +# `ably-pubsub-core==` exactly, so a server release without its +# core is uninstallable and a core release without its server is invisible. +# +# PRE-FLIGHT. Everything that can be checked is checked before the first +# upload, by scripts/release_preflight.py — the same script check.yml's +# release-dry-run job runs on every pull request. It verifies the release +# version against every version site (both pyproject `version` fields, the +# core's `lib_version`, the server's `__version__`) and the server's exact core +# pins, that dist/ holds exactly one wheel and one sdist per distribution, that +# the core artifacts carry the generated `ably_pubsub/core/sync/` flavour, that +# neither wheel ships `ably_pubsub/__init__.py` (the namespace must stay PEP +# 420) or any file the other wheel also ships, and that `twine check` passes. +# +# CORE BEFORE SERVER. The two publish steps are ordered, so the server is never +# visible on the index before the core version it pins. +# +# PARTIAL RELEASES ARE RE-RUNNABLE, NOT IMPOSSIBLE. PyPI has no cross-project +# transaction: two projects means two uploads, and the second can fail after +# the first succeeded. Both steps therefore set `skip-existing: true`, so +# re-running this workflow at the same version skips whatever already landed +# and completes the release. Never bump the version to work around a partial +# release — re-run it. +# +# TRUSTED PUBLISHING. Both projects must have a trusted publisher configured on +# pypi.org (and on test.pypi.org) bound to this repository, this workflow file +# (`release.yml`) and the `pypi` / `testpypi` environment respectively — plan +# step 16, done immediately after the repo rename so the binding is made once +# against the new name. PyPI's OIDC token covers every project that trusts the +# requesting configuration, so one job's `id-token: write` publishes both. +# The `pypi` environment's required-reviewer rule is the human approval gate. +# +# TRIGGERS. A `v` tag push releases to PyPI. A manual dispatch always +# goes to TestPyPI and only reaches PyPI when `publish` is set — which is how +# prereleases are cut from a branch (`gh workflow run release.yml --ref +# -f version=4.0.0rc1 -f publish=true`). `workflow_dispatch` only +# works for workflow files present on the default branch. +name: Publish Python distributions to PyPI on: workflow_dispatch: + inputs: + version: + description: 'Version to release, e.g. 4.0.0 or 4.0.0rc1 — must equal every version site and the server''s core pin' + required: true + type: string + publish: + description: 'Also publish to PyPI (not just TestPyPI)' + required: false + default: false + type: boolean push: tags: - 'v[0-9]+.[0-9]+.[0-9]+*' @@ -10,16 +60,33 @@ permissions: {} jobs: build: - name: Build distribution 📦 + name: Build distributions 📦 runs-on: ubuntu-latest permissions: contents: read + outputs: + version: ${{ steps.release-version.outputs.version }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: submodules: 'recursive' persist-credentials: false + + - name: Determine the release version + id: release-version + env: + VERSION_INPUT: ${{ inputs.version }} + run: | + set -euo pipefail + if [ -n "${VERSION_INPUT}" ]; then + RELEASE_VERSION="${VERSION_INPUT}" + else + RELEASE_VERSION="${GITHUB_REF#refs/tags/v}" + fi + echo "Releasing ${RELEASE_VERSION}" + echo "version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + - name: Set up Python 3.12 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 id: setup-python @@ -32,49 +99,49 @@ jobs: enable-cache: false - name: Install dependencies - run: uv sync --extra crypto --extra dev - - name: Generate rest sync code and tests + run: uv sync + + # Cheap checks that need no build: the version sites, and — first of all + # — that this ref actually has the split layout. This workflow file also + # lives on `main` (workflow_dispatch only offers workflows present on the + # default branch), where it is inert: dispatched there, the pre-flight + # stops here with an explanation instead of a confusing build error. + - name: 'Pre-flight: layout and version sites (before anything is built)' + env: + RELEASE_VERSION: ${{ steps.release-version.outputs.version }} + run: uv run python scripts/release_preflight.py --version "$RELEASE_VERSION" + + - name: Generate the sync flavour run: uv run unasync - - name: Build a binary wheel and a source tarball - run: uv build + - name: Build both distributions into one dist/ + run: | + set -euo pipefail + uv build --package ably-pubsub-core --out-dir dist + uv build --package ably-pubsub-server --out-dir dist + + # Nothing below this point is reversible, so this is the last chance to + # refuse the release. Same script as check.yml's release-dry-run job. + - name: 'Pre-flight: nothing is uploaded unless everything agrees' + env: + RELEASE_VERSION: ${{ steps.release-version.outputs.version }} + run: uv run python scripts/release_preflight.py --version "$RELEASE_VERSION" dist/ + - name: Store the distribution packages uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: python-package-distributions path: dist/ - - name: Check that wheel and tarball contains ably/sync/ - run: | - # Check wheel - WHEEL=$(ls dist/*.whl | head -n 1) - echo "Checking wheel: $WHEEL" - if unzip -l "$WHEEL" | grep -q "ably/sync/"; then - echo "✅ Found ably/sync/ in wheel" - else - unzip -l "$WHEEL" - echo "❌ ably/sync/ not found in wheel" - exit 1 - fi - - # Check tarball - TARBALL=$(ls dist/*.tar.gz | head -n 1) - echo "Checking tarball: $TARBALL" - if tar -tzf "$TARBALL" | grep -q "ably/sync/"; then - echo "✅ Found ably/sync/ in tarball" - else - tar -tzf "$TARBALL" - echo "❌ ably/sync/ not found in tarball" - exit 1 - fi - publish-to-pypi: - name: Publish Python distribution to PyPI - if: startsWith(github.ref, 'refs/tags/v') # only publish to PyPI on tag pushes + publish-to-testpypi: + name: Publish distributions 📦 to TestPyPI needs: - build runs-on: ubuntu-latest + environment: - name: pypi - url: https://pypi.org/p/ably + name: testpypi + url: https://test.pypi.org/p/ably-pubsub-server + permissions: id-token: write # IMPORTANT: mandatory for trusted publishing @@ -85,41 +152,41 @@ jobs: name: python-package-distributions path: dist/ - - name: Extract tag - id: tag + # pypa/gh-action-pypi-publish uploads a whole directory, so the two + # projects are split into two directories to be uploaded in order. + - name: Split the dists by project run: | - TAG=${GITHUB_REF#refs/tags/v} - echo "tag=$TAG" >> $GITHUB_OUTPUT + set -euo pipefail + mkdir -p dist-core dist-server + mv dist/ably_pubsub_core-* dist-core/ + mv dist/ably_pubsub_server-* dist-server/ - - name: Read VERSION_NAME from dist/ - id: version - run: | - VERSION_NAME=$(basename dist/ably-*.tar.gz | sed -E 's/^ably-([^-]+)\.tar\.gz$/\1/') - echo "version=$VERSION_NAME" >> $GITHUB_OUTPUT - - - name: Compare version with tag - run: | - if [ "$VERSION" != "$TAG" ]; then - echo "VERSION ($VERSION) does not match tag ($TAG)." - exit 1 - fi - env: - VERSION: ${{ steps.version.outputs.version }} - TAG: ${{ steps.tag.outputs.tag }} + - name: Publish ably-pubsub-core 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + with: + packages-dir: dist-core/ + repository-url: https://test.pypi.org/legacy/ + skip-existing: true - - name: Publish distribution 📦 to PyPI + - name: Publish ably-pubsub-server 📦 to TestPyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + with: + packages-dir: dist-server/ + repository-url: https://test.pypi.org/legacy/ + skip-existing: true - publish-to-testpypi: - name: Publish Python distribution to TestPyPI + publish-to-pypi: + name: Publish distributions 📦 to PyPI + # Tag pushes release; a manual dispatch has to opt in explicitly. + if: startsWith(github.ref, 'refs/tags/v') || inputs.publish + # Deliberately not `needs: publish-to-testpypi`: TestPyPI is a staging + # signal, not a gate — an outage there must not block a real release. needs: - build runs-on: ubuntu-latest - environment: - name: testpypi - url: https://test.pypi.org/p/ably - + name: pypi + url: https://pypi.org/p/ably-pubsub-server permissions: id-token: write # IMPORTANT: mandatory for trusted publishing @@ -129,7 +196,24 @@ jobs: with: name: python-package-distributions path: dist/ - - name: Publish distribution 📦 to TestPyPI + + - name: Split the dists by project + run: | + set -euo pipefail + mkdir -p dist-core dist-server + mv dist/ably_pubsub_core-* dist-core/ + mv dist/ably_pubsub_server-* dist-server/ + + # Core first: the server pins this exact version, so it must never be the + # one that is visible on the index alone. + - name: Publish ably-pubsub-core 📦 to PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: - repository-url: https://test.pypi.org/legacy/ + packages-dir: dist-core/ + skip-existing: true + + - name: Publish ably-pubsub-server 📦 to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + with: + packages-dir: dist-server/ + skip-existing: true diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py new file mode 100644 index 00000000..f8723862 --- /dev/null +++ b/scripts/release_preflight.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Pre-flight checks for a lockstep release of ably-pubsub-core and ably-pubsub-server. + +Both distributions are released at one version and the server pins the core +exactly, so a release is only meaningful if every version site agrees and both +sets of artifacts are well formed. PyPI has no cross-project transaction: once +the core is uploaded it cannot be taken back, so everything that can be checked +must be checked *before* the first upload. + +This is the single implementation behind both callers: + + # release.yml — the version comes from the tag or the dispatch input + uv run python scripts/release_preflight.py --version 4.0.0 dist/ + + # check.yml release-dry-run — no authoritative version, just internal agreement + uv run python scripts/release_preflight.py dist/ + + # /release skill and local use — version sites only, no build needed + uv run python scripts/release_preflight.py + +Exit status is 0 when every check passes and 1 otherwise, with every failure +reported (the run does not stop at the first one). +""" + +import argparse +import re +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +CORE_PYPROJECT = REPO_ROOT / 'core' / 'pyproject.toml' +SERVER_PYPROJECT = REPO_ROOT / 'server' / 'pyproject.toml' +CORE_INIT = REPO_ROOT / 'core' / 'src' / 'ably_pubsub' / 'core' / '__init__.py' +SERVER_INIT = REPO_ROOT / 'server' / 'src' / 'ably_pubsub' / 'server' / '__init__.py' + +CORE_DIST = 'ably_pubsub_core' +SERVER_DIST = 'ably_pubsub_server' + +# PEP 440, the subset a release can actually be: a release segment, an optional +# pre/post/dev suffix. Local versions are excluded — PyPI rejects them. +PEP_440 = re.compile( + r'^([1-9][0-9]*!)?' + r'(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*' + r'((a|b|rc)(0|[1-9][0-9]*))?' + r'(\.post(0|[1-9][0-9]*))?' + r'(\.dev(0|[1-9][0-9]*))?$' +) + + +def require_split_layout(): + """Refuse to run against a tree that does not have the split layout. + + This script (and the release workflow that calls it) also exists on `main`, + because `workflow_dispatch` only offers a workflow that is present on the + default branch — see the plan's step 15b. `main` still has the single flat + `ably/` package, so a dispatch there must stop here with an explanation + rather than a FileNotFoundError from the first version site it reads. + + On the split layout every path below exists, so this is a no-op. + """ + missing = [ + str(path.relative_to(REPO_ROOT)) + for path in (CORE_PYPROJECT, SERVER_PYPROJECT) + if not path.is_file() + ] + if missing: + raise SystemExit( + 'pre-flight: this workflow releases the split distributions ' + '(ably-pubsub-core and ably-pubsub-server), but this ref still has the ' + 'single `ably` layout — no ' + ' or '.join(missing) + '. There is nothing ' + 'here to release in lockstep. Dispatch this workflow against a ref that ' + 'has the split layout (--ref integration/v4, or a pubsub-split/* branch); ' + 'releases of the legacy `ably` distribution are cut from its maintenance ' + 'branch with that branch\'s own single-distribution release.yml.' + ) + + +class Failures: + def __init__(self): + self.messages = [] + + def add(self, message): + self.messages.append(message) + + def check(self, condition, message): + if not condition: + self.add(message) + return bool(condition) + + +def read_scalar(path, pattern, what): + """Read a single quoted scalar out of a file. + + Deliberately regex rather than tomllib/import: tomllib only arrived in 3.11, + and importing the packages would make the pre-flight depend on them being + installed rather than on what is actually in the tree about to be released. + """ + match = re.search(pattern, path.read_text(), re.MULTILINE) + if not match: + raise SystemExit(f'pre-flight could not read {what} from {path}') + return match.group(1) + + +def version_sites(): + return { + f'{CORE_PYPROJECT.relative_to(REPO_ROOT)} [project] version': + read_scalar(CORE_PYPROJECT, r'^version = "([^"]+)"', 'version'), + f'{SERVER_PYPROJECT.relative_to(REPO_ROOT)} [project] version': + read_scalar(SERVER_PYPROJECT, r'^version = "([^"]+)"', 'version'), + f'{CORE_INIT.relative_to(REPO_ROOT)} lib_version': + read_scalar(CORE_INIT, r"^lib_version = '([^']+)'", 'lib_version'), + f'{SERVER_INIT.relative_to(REPO_ROOT)} __version__': + read_scalar(SERVER_INIT, r"^__version__ = '([^']+)'", '__version__'), + } + + +def core_pins(): + """Every `ably-pubsub-core[...]==` pin in the server's pyproject.""" + text = SERVER_PYPROJECT.read_text() + return set(re.findall(r'"ably-pubsub-core(?:\[[\w,]+\])?==([^"]+)"', text)) + + +def core_extras(): + """The names in the core's `[project.optional-dependencies]` table.""" + text = CORE_PYPROJECT.read_text() + section = text.split('[project.optional-dependencies]', 1)[1] + section = re.split(r'^\[', section, maxsplit=1, flags=re.MULTILINE)[0] + return re.findall(r'^(\w+) = \[', section, re.MULTILINE) + + +def check_versions(failures, expected): + sites = version_sites() + pins = core_pins() + + if expected is None: + # No authoritative version: the sites only have to agree with each other, + # and the first one is then taken as the release version for the rest. + expected = sorted(sites.values())[0] + + failures.check( + PEP_440.match(expected), + f'version {expected!r} is not a valid PEP 440 public version', + ) + + for site, value in sorted(sites.items()): + failures.check(value == expected, f'{site} is {value!r}, expected {expected!r}') + + failures.check(pins, 'no ably-pubsub-core== pin found in server/pyproject.toml') + for pin in sorted(pins): + failures.check( + pin == expected, + f'server/pyproject.toml pins ably-pubsub-core=={pin}, expected =={expected}', + ) + + # Every extra the core offers must be forwarded, or `pip install + # ably-pubsub-server[crypto]` silently installs a core without the extra. + server_text = SERVER_PYPROJECT.read_text() + for extra in core_extras(): + failures.check( + f'"ably-pubsub-core[{extra}]=={expected}"' in server_text, + f'server/pyproject.toml does not forward the core extra {extra!r} at {expected}', + ) + + return expected + + +def one_of_each(failures, dist_dir, version): + """Exactly one wheel and one sdist per distribution, at the release version.""" + found = {} + for name in (CORE_DIST, SERVER_DIST): + wheels = sorted(dist_dir.glob(f'{name}-*.whl')) + sdists = sorted(dist_dir.glob(f'{name}-*.tar.gz')) + failures.check(len(wheels) == 1, f'expected exactly one {name} wheel in {dist_dir}, found {len(wheels)}') + failures.check(len(sdists) == 1, f'expected exactly one {name} sdist in {dist_dir}, found {len(sdists)}') + for path in wheels + sdists: + failures.check( + f'-{version}' in path.name.replace(f'{name}', '', 1), + f'{path.name} is not version {version}', + ) + found[name] = (wheels[0] if wheels else None, sdists[0] if sdists else None) + + unexpected = sorted( + p.name for p in dist_dir.iterdir() + if p.suffix in {'.whl', '.gz'} and not p.name.startswith((CORE_DIST + '-', SERVER_DIST + '-')) + ) + failures.check(not unexpected, f'unexpected files in {dist_dir}: {", ".join(unexpected)}') + return found + + +def wheel_names(path): + with zipfile.ZipFile(path) as zf: + return set(zf.namelist()) + + +def sdist_names(path): + with tarfile.open(path) as tf: + # Strip the `-/` prefix so paths compare like the wheel's. + return {n.split('/', 1)[1] for n in tf.getnames() if '/' in n} + + +def check_artifacts(failures, dist_dir, version): + found = one_of_each(failures, dist_dir, version) + + core_wheel, core_sdist = found[CORE_DIST] + server_wheel, server_sdist = found[SERVER_DIST] + + # The generated sync flavour is not in git; a core built without running + # unasync first looks fine until someone imports ably_pubsub.core.sync. + if core_wheel: + names = wheel_names(core_wheel) + failures.check( + any(n.startswith('ably_pubsub/core/sync/') for n in names), + f'{core_wheel.name} does not contain ably_pubsub/core/sync/ (was unasync run?)', + ) + if core_sdist: + names = sdist_names(core_sdist) + failures.check( + any('ably_pubsub/core/sync/' in n for n in names), + f'{core_sdist.name} does not contain ably_pubsub/core/sync/ (was unasync run?)', + ) + + # PEP 420: an __init__.py here would belong to whichever distribution shipped + # it, so removing that one would delete the other's entry into the namespace. + for wheel in (core_wheel, server_wheel): + if wheel: + failures.check( + 'ably_pubsub/__init__.py' not in wheel_names(wheel), + f'{wheel.name} ships ably_pubsub/__init__.py; the namespace must stay PEP 420', + ) + + # Two wheels writing the same path into site-packages means installing one + # overwrites the other and uninstalling either breaks what is left. + if core_wheel and server_wheel: + core_files = {n for n in wheel_names(core_wheel) if not n.endswith('/')} + server_files = {n for n in wheel_names(server_wheel) if not n.endswith('/')} + overlap = sorted(core_files & server_files) + failures.check(not overlap, f'the two wheels both ship: {", ".join(overlap)}') + + # Both distributions must carry the licence they claim in their metadata. + for path in (core_wheel, server_wheel): + if path: + failures.check( + any(n.endswith('.dist-info/licenses/LICENSE') for n in wheel_names(path)), + f'{path.name} does not bundle LICENSE', + ) + for path in (core_sdist, server_sdist): + if path: + failures.check('LICENSE' in sdist_names(path), f'{path.name} does not bundle LICENSE') + + artifacts = sorted( + str(p) for p in dist_dir.iterdir() + if p.is_file() and (p.name.endswith('.whl') or p.name.endswith('.tar.gz')) + ) + result = subprocess.run( + [sys.executable, '-m', 'twine', 'check', *artifacts], + capture_output=True, + text=True, + ) + sys.stdout.write(result.stdout) + sys.stdout.write(result.stderr) + failures.check(result.returncode == 0, 'twine check failed') + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + '--version', + help='the version being released (from the tag or the workflow_dispatch input). ' + 'Without it the version sites only have to agree with each other.', + ) + parser.add_argument( + 'dist', + nargs='?', + type=Path, + help='directory holding the built distributions. Omit to check the version sites only.', + ) + args = parser.parse_args(argv) + + require_split_layout() + + failures = Failures() + version = check_versions(failures, args.version) + + if args.dist is not None: + if not args.dist.is_dir(): + raise SystemExit(f'pre-flight: {args.dist} is not a directory') + check_artifacts(failures, args.dist, version) + + if failures.messages: + print(f'\nPre-flight FAILED ({len(failures.messages)} problem(s)); nothing should be published:\n', + file=sys.stderr) + for message in failures.messages: + print(f' - {message}', file=sys.stderr) + return 1 + + scope = 'version sites' if args.dist is None else f'version sites and the artifacts in {args.dist}' + print(f'Pre-flight OK: {scope} all agree on {version}.') + return 0 + + +if __name__ == '__main__': + sys.exit(main())