diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fe6f14e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +# The workflow actions are pinned to commit SHAs, which never move on +# their own; this keeps them current. Dependabot reads the version from +# the trailing comment on each pin and rewrites both together. The conda +# pins are kept current by the Conda update workflow instead. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + # A compromised release is usually caught and pulled within days, so + # hold off on proposing one until it has been out for a week. + cooldown: + default-days: 7 + # One pull request per month rather than one per action + groups: + actions: + patterns: ["*"] diff --git a/.github/scripts/conda_update.py b/.github/scripts/conda_update.py new file mode 100644 index 0000000..8696ced --- /dev/null +++ b/.github/scripts/conda_update.py @@ -0,0 +1,253 @@ +"""Move the conda pins to the newest versions that solve together. + +Each exact pin in ``environment.yml`` is loosened to a lower bound at its +current version, and micromamba solves the result, without installing it, +for every platform the CI builds on. The versions it picks are written back as +exact pins, so the file only ever holds a set the solver has accepted as +a whole. Bumping each pin on its own, as Dependabot does, misses the +constraints packages place on each other. + +Each platform is solved separately, and conda-forge builds reach each +platform at different times, so the platforms can choose different +versions. Where they disagree, the package is capped at each version +chosen and every platform solves again. Caps only ever fall and the +current pins always satisfy them, so the solves settle on one set that +installs everywhere, at worst the set already in the file. + +Wildcard pins reach the solver unchanged, which makes them the way to +hold a package back. A lower bound also admits development snapshots, +which conda orders above the last release before them; when the solver +picks one for a pinned package, the solve runs again with that package +capped below it. Each change is printed as ``name old -> new``. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +logger = logging.getLogger(__name__) + +# The environment file at the repository root, the only one with pins +SOURCE = Path(__file__).parents[2] / "environment.yml" + +# An exact pin, the only kind that moves; a trailing comment is kept +EXACT = re.compile( + r"^ - (?P[A-Za-z0-9_.-]+)=(?P[0-9][A-Za-z0-9_.+]*)" + r"(?P\s*(?:#.*)?)$" +) + +# Marks a development snapshot or a release candidate in a version +PRERELEASE = re.compile(r"dev|rc|alpha|beta|pre|\d[ab]\d", re.IGNORECASE) + +# The platforms of the two CI build jobs. The compiler metapackages resolve +# to gcc or clang per platform, so both solve the same file. +PLATFORMS = ("linux-64", "osx-arm64") + +# The macOS of the test runner. A Linux host reports none, and without one +# the solver rejects every build that needs a minimum macOS. +MACOS_VERSION = "26.0" + + +def loosen(source: str, bounds: dict[str, set[str]]) -> str: + """ + Return the source with every exact pin made a lower bound. + + A package in ``bounds`` also carries each upper bound it maps to, such + as ``<=12.1.0``; the solver reads several as the tightest of them. + """ + lines = [] + for line in source.splitlines(): + # An exact pin becomes a floor the solver may move up from, plus the + # caps earlier passes added; wildcards and the header pass through as + # written. The trailing comment is dropped, as the solver never sees + # this file again. + match = EXACT.match(line) + if match is not None: + line = f" - {match['name']}>={match['version']}" + line += "".join( + f",{bound}" for bound in sorted(bounds.get(match["name"], ())) + ) + lines.append(line) + return "\n".join(lines) + "\n" + + +def pinned(source: str) -> dict[str, str]: + """Return the exact pins in the source, each name mapped to its version.""" + pins = (EXACT.match(line) for line in source.splitlines()) + return {m["name"]: m["version"] for m in pins if m is not None} + + +def solve(environment: str, platform: str) -> dict[str, str]: + """ + Solve an environment file for a platform without installing it. + + Returns the version chosen for every package in the solution, keyed by + name. Raises RuntimeError with the solver's output when it fails. + """ + # CI has micromamba and a developer machine usually mamba; both share + # the same solver and the same JSON output + solver = shutil.which("micromamba") or shutil.which("mamba") + if solver is None: + raise RuntimeError("neither micromamba nor mamba is on PATH") + + with tempfile.TemporaryDirectory() as scratch: + # The solver reads the file type from the extension + spec = Path(scratch) / "environment.yml" + spec.write_text(environment) + + # A dry run solves and reports without downloading or installing; the + # prefix only has to be a path that does not exist yet + command = [solver, "create", "--dry-run", "--json", "-y"] + command += ["-p", str(Path(scratch) / "prefix"), "-f", str(spec)] + command += ["--platform", platform] + + # A macOS solve needs a macOS version to check builds against + env = os.environ.copy() + if platform.startswith("osx"): + env["CONDA_OVERRIDE_OSX"] = MACOS_VERSION + started = time.monotonic() + result = subprocess.run(command, capture_output=True, text=True, env=env) + logger.info(" %s solved in %.0fs", platform, time.monotonic() - started) + + # An unsatisfiable set fails here, with the solver's own explanation + if result.returncode != 0: + raise RuntimeError(f"the solve failed:\n{result.stdout}{result.stderr}") + + # LINK lists every package the environment would contain, dependencies + # included, each with the version the solver chose + linked = json.loads(result.stdout)["actions"]["LINK"] + return {package["name"]: package["version"] for package in linked} + + +def apply(source: str, solved: dict[str, str]) -> tuple[str, list[str]]: + """ + Write the solved versions into the exact pins of the source. + + Returns the new source and one ``name old -> new`` line per change. + Raises ValueError for a pinned package the solution does not contain. + """ + lines, changes = [], [] + for line in source.splitlines(): + # Only exact pins are rewritten; every other line is copied as it is + match = EXACT.match(line) + if match is not None: + name, old = match["name"], match["version"] + + # Every pinned package is a direct request, so a solution without + # it means the solve and the file disagree + if name not in solved: + raise ValueError(f"{name} is pinned but not in the solution") + + # Replace the version alone, keeping any trailing comment + if solved[name] != old: + line = f" - {name}={solved[name]}{match['rest']}" + changes.append(f"{name} {old} -> {solved[name]}") + lines.append(line) + return "\n".join(lines) + "\n", changes + + +def tighten( + solutions: dict[str, dict[str, str]], current: dict[str, str] +) -> set[tuple[str, str]]: + """ + Return the upper bounds that bring the platforms towards one solution. + + Each bound is a ``(name, bound)`` pair. An empty result means no pinned + package is on a new pre-release and every platform chose the same + versions. + """ + # A pinned package the solver moved to a pre-release, on any platform, is + # capped just below it. A pin that is already a pre-release is left alone, + # as a cap at its own version could never be met. + snapshots = set() + for platform, solution in solutions.items(): + for name, version in current.items(): + chosen = solution.get(name, version) + if chosen != version and PRERELEASE.search(chosen): + logger.info( + " %s: %s chose pre-release %s, capping below it", + name, + platform, + chosen, + ) + snapshots.add((name, f"<{chosen}")) + if snapshots: + return snapshots + + # A pinned package the platforms disagree on is capped at every version + # chosen, which holds each platform to the lowest of them. A package a + # platform does not install is judged on the platforms that do. + caps = set() + for name in current: + versions = {p: s[name] for p, s in solutions.items() if name in s} + if len(set(versions.values())) > 1: + logger.info( + " %s: platforms disagree (%s), capping at the lowest", + name, + ", ".join(f"{p} {v}" for p, v in versions.items()), + ) + caps |= {(name, f"<={version}") for version in versions.values()} + return caps + + +def main() -> None: + argparse.ArgumentParser(description=__doc__.splitlines()[0]).parse_args() + + # Progress goes to stderr, leaving stdout to the changes alone + logging.basicConfig(level=logging.INFO, format="%(message)s") + + source = SOURCE.read_text() + current = pinned(source) + bounds: dict[str, set[str]] = {} + rounds = 0 + try: + while True: + rounds += 1 + caps = sum(len(b) for b in bounds.values()) + logger.info( + "Round %d: solving %d platforms, %d caps", rounds, len(PLATFORMS), caps + ) + # Solve every platform from the same loosened file + loosened = loosen(source, bounds) + solutions = { + platform: solve(loosened, platform) for platform in PLATFORMS + } + + # Stop once the platforms agree. Every new bound excludes a version + # some platform just chose, so each pass makes progress and the + # loop ends; a bound already in place would mean it cannot. + tighter = tighten(solutions, current) + if not tighter: + break + if all(bound in bounds.get(name, ()) for name, bound in tighter): + raise RuntimeError("the platforms cannot agree on one set of pins") + for name, bound in tighter: + bounds.setdefault(name, set()).add(bound) + + # The platforms agree, so any one of them gives each pinned version + solved: dict[str, str] = {} + for solution in solutions.values(): + solved |= solution + updated, changes = apply(source, solved) + logger.info("Agreed after %d rounds, %d pins changed", rounds, len(changes)) + except (RuntimeError, ValueError) as error: + sys.exit(f"{SOURCE}: {error}") + + # The file is only touched once every step has succeeded, and the + # changes go to stdout for the pull request body + SOURCE.write_text(updated) + print("\n".join(changes)) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 8765d03..a27defe 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -1,4 +1,10 @@ -on: [push, pull_request] +on: + push: + branches: ["main"] + pull_request: null + workflow_dispatch: null +permissions: + contents: read jobs: build_test: name: Build/Test @@ -14,27 +20,26 @@ jobs: # fresh conda install may not preserve, so compare contents instead. CCACHE_COMPILERCHECK: content steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # The runner context is not available in the job env, so the cache # directories are set here for every later step. - name: Set cache directories run: | echo "CCACHE_DIR=${RUNNER_TEMP}/ccache" >> "$GITHUB_ENV" echo "DEPS_DIR=${RUNNER_TEMP}/deps" >> "$GITHUB_ENV" - - uses: conda-incubator/setup-miniconda@v4 + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 with: channels: conda-forge conda-remove-defaults: true - name: Install dependencies run: | - conda env update --file requirements.txt --name test - conda install --yes --name test ccache + conda env update --file environment.yml --name test conda list # Each run restores the newest cache under the prefix and saves its # own under the commit, so the cache rolls forward rather than # freezing at the first save. - name: Restore ccache - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ github.sha }} @@ -45,10 +50,10 @@ jobs: # so the key covers the file that pins them and the one that chooses # the toolchain. - name: Restore dependency sources - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ env.DEPS_DIR }} - key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }} + key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'environment.yml') }} # List each restored dependency with the commit it sits at. The # directory starts empty on every job, so anything in it came from # the cache. A login shell runs ~/.bash_logout on exit, which can fail diff --git a/.github/workflows/conda-update.yml b/.github/workflows/conda-update.yml new file mode 100644 index 0000000..41c1b82 --- /dev/null +++ b/.github/workflows/conda-update.yml @@ -0,0 +1,74 @@ +name: Conda update +# Moves the conda pins to the newest versions that solve together on every +# platform the CI builds on, and opens one pull request with the result. +# .github/scripts/conda_update.py does the solving. +on: + schedule: + # 06:36 UTC on the 4th of each month, away from the start of the hour + # and the start of the month + - cron: "36 6 4 * *" + workflow_dispatch: null +permissions: + contents: read +# A manual run alongside the scheduled one would race it to the branch +concurrency: + group: conda-update + cancel-in-progress: false +jobs: + update: + runs-on: ubuntu-latest + permissions: + contents: write # push the update branch + pull-requests: write # open or edit the pull request + actions: write # start CI on the branch + env: + BRANCH: conda-update + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # With no environment named, only the micromamba binary is installed; + # the script runs its own dry-run solves + - uses: mamba-org/setup-micromamba@f457c30a868e4760d3a6fcea5f25dc655b8edf39 # v3.2.1 + # Progress goes to the log and the list of changes to a file, for the + # pull request body + - name: Solve the pins + run: python3 .github/scripts/conda_update.py > "$RUNNER_TEMP/changes.txt" + - name: Push the update branch + id: push + run: | + if git diff --quiet; then + echo "Every pin is already the newest that solves" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$BRANCH" + git commit -qam "Update the conda pins" + # The checkout keeps no credentials; gh lends the job token to git + gh auth setup-git + git push --force origin "$BRANCH" + echo "pushed=true" >> "$GITHUB_OUTPUT" + - name: Open or update the pull request + if: steps.push.outputs.pushed == 'true' + run: | + { + echo "The newest versions that solve together on linux-64 and" + echo "osx-arm64, from \`.github/scripts/conda_update.py\`:" + echo + echo '```' + cat "$RUNNER_TEMP/changes.txt" + echo '```' + } > "$RUNNER_TEMP/body.md" + if [ "$(gh pr view "$BRANCH" --json state --jq .state 2>/dev/null)" = "OPEN" ]; then + gh pr edit "$BRANCH" --body-file "$RUNNER_TEMP/body.md" + else + gh pr create --base main --head "$BRANCH" \ + --title "Update the conda pins" --body-file "$RUNNER_TEMP/body.md" + fi + # A push or pull request made with the job token starts no workflows, + # but a dispatch does, and its checks attach to the branch's commit + - name: Start CI on the branch + if: steps.push.outputs.pushed == 'true' + run: gh workflow run build_test.yml --ref "$BRANCH" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ec222c4..4c774f6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,10 +1,12 @@ name: Docs -on: [push, pull_request] -# Allow the deploy job to publish to GitHub Pages. +on: + push: + branches: ["main"] + pull_request: null + workflow_dispatch: null +# Read-only by default; only the deploy job can publish. permissions: contents: read - pages: write - id-token: write # Cancel superseded runs on the same ref. concurrency: group: docs-${{ github.ref }} @@ -14,12 +16,12 @@ jobs: name: Build docs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Doxygen only parses the source, so no C++ toolchain or the # HDF5/gemmi/Eigen dependencies are needed here. - name: Install Doxygen and Graphviz run: sudo apt-get update && sudo apt-get install -y doxygen graphviz - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - name: Install Python dependencies @@ -30,13 +32,13 @@ jobs: # artifact so reviewers can preview the docs without building. - name: Upload preview artifact if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: docs-html path: docs/_build/html - name: Upload Pages artifact if: github.ref == 'refs/heads/main' && github.event_name == 'push' - uses: actions/upload-pages-artifact@v5 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: docs/_build/html deploy: @@ -45,10 +47,14 @@ jobs: if: github.ref == 'refs/heads/main' && github.event_name == 'push' needs: build runs-on: ubuntu-latest + # Publishing to Pages authenticates with an OIDC token + permissions: + pages: write + id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - name: Deploy id: deployment - uses: actions/deploy-pages@v5 + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/README.md b/README.md index 9c89787..78b72ad 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Quick Start -- Ensure you have an environment with CMake and compilers available (see requirements.txt) +- Ensure you have an environment with CMake and compilers available (see environment.yml) - Dependencies: - HDF5 (Required) - gemmi (Required) diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..7bdb298 --- /dev/null +++ b/environment.yml @@ -0,0 +1,17 @@ +name: dx2 +channels: + - conda-forge +# Packages are pinned exactly so that a build only changes when this +# file does. The Conda update workflow proposes the bumps and leaves +# wildcard pins alone. +dependencies: + - c-compiler=2.0.0 + - cxx-compiler=2.0.0 + - ccache=4.14.1 + - gemmi=0.7.5 + - cmake=4.4.5 + - hdf5=1.14.* # Held on 1.14; the move to 2.x is a migration made by hand + - hdf5-external-filter-plugins=0.1.0 + - nlohmann_json=3.12.0 + - fmt=12.2.0 + - eigen=5.0.1 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 864553f..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -c-compiler -cxx-compiler -gemmi -cmake -hdf5=1.14.6 -hdf5-external-filter-plugins=0.1.0 -nlohmann_json>=3.11.3 -fmt>=11.0 -eigen>=3.4