Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# The workflow actions are pinned to commit SHAs, which never move on
# their own; this keeps them current. Dependabot reads the version from
# the trailing comment on each pin and rewrites both together. The conda
# pins are kept current by the Conda update workflow instead.
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: monthly
# A compromised release is usually caught and pulled within days, so
# hold off on proposing one until it has been out for a week.
cooldown:
default-days: 7
# One pull request per month rather than one per action
groups:
actions:
patterns: ["*"]
253 changes: 253 additions & 0 deletions .github/scripts/conda_update.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@
"""Move the conda pins to the newest versions that solve together.

Each exact pin in ``environment.yml`` is loosened to a lower bound at its
current version, and micromamba solves the result, without installing it,
for every platform the CI builds on. The versions it picks are written back as
exact pins, so the file only ever holds a set the solver has accepted as
a whole. Bumping each pin on its own, as Dependabot does, misses the
constraints packages place on each other.

Each platform is solved separately, and conda-forge builds reach each
platform at different times, so the platforms can choose different
versions. Where they disagree, the package is capped at each version
chosen and every platform solves again. Caps only ever fall and the
current pins always satisfy them, so the solves settle on one set that
installs everywhere, at worst the set already in the file.

Wildcard pins reach the solver unchanged, which makes them the way to
hold a package back. A lower bound also admits development snapshots,
which conda orders above the last release before them; when the solver
picks one for a pinned package, the solve runs again with that package
capped below it. Each change is printed as ``name old -> new``.
"""

from __future__ import annotations

import argparse
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path

logger = logging.getLogger(__name__)

# The environment file at the repository root, the only one with pins
SOURCE = Path(__file__).parents[2] / "environment.yml"

# An exact pin, the only kind that moves; a trailing comment is kept
EXACT = re.compile(
r"^ - (?P<name>[A-Za-z0-9_.-]+)=(?P<version>[0-9][A-Za-z0-9_.+]*)"
r"(?P<rest>\s*(?:#.*)?)$"
)

# Marks a development snapshot or a release candidate in a version
PRERELEASE = re.compile(r"dev|rc|alpha|beta|pre|\d[ab]\d", re.IGNORECASE)

# The platforms of the two CI build jobs. The compiler metapackages resolve
# to gcc or clang per platform, so both solve the same file.
PLATFORMS = ("linux-64", "osx-arm64")

# The macOS of the test runner. A Linux host reports none, and without one
# the solver rejects every build that needs a minimum macOS.
MACOS_VERSION = "26.0"


def loosen(source: str, bounds: dict[str, set[str]]) -> str:
"""
Return the source with every exact pin made a lower bound.

A package in ``bounds`` also carries each upper bound it maps to, such
as ``<=12.1.0``; the solver reads several as the tightest of them.
"""
lines = []
for line in source.splitlines():
# An exact pin becomes a floor the solver may move up from, plus the
# caps earlier passes added; wildcards and the header pass through as
# written. The trailing comment is dropped, as the solver never sees
# this file again.
match = EXACT.match(line)
if match is not None:
line = f" - {match['name']}>={match['version']}"
line += "".join(
f",{bound}" for bound in sorted(bounds.get(match["name"], ()))
)
lines.append(line)
return "\n".join(lines) + "\n"


def pinned(source: str) -> dict[str, str]:
"""Return the exact pins in the source, each name mapped to its version."""
pins = (EXACT.match(line) for line in source.splitlines())
return {m["name"]: m["version"] for m in pins if m is not None}


def solve(environment: str, platform: str) -> dict[str, str]:
"""
Solve an environment file for a platform without installing it.

Returns the version chosen for every package in the solution, keyed by
name. Raises RuntimeError with the solver's output when it fails.
"""
# CI has micromamba and a developer machine usually mamba; both share
# the same solver and the same JSON output
solver = shutil.which("micromamba") or shutil.which("mamba")
if solver is None:
raise RuntimeError("neither micromamba nor mamba is on PATH")

with tempfile.TemporaryDirectory() as scratch:
# The solver reads the file type from the extension
spec = Path(scratch) / "environment.yml"
spec.write_text(environment)

# A dry run solves and reports without downloading or installing; the
# prefix only has to be a path that does not exist yet
command = [solver, "create", "--dry-run", "--json", "-y"]
command += ["-p", str(Path(scratch) / "prefix"), "-f", str(spec)]
command += ["--platform", platform]

# A macOS solve needs a macOS version to check builds against
env = os.environ.copy()
if platform.startswith("osx"):
env["CONDA_OVERRIDE_OSX"] = MACOS_VERSION
started = time.monotonic()
result = subprocess.run(command, capture_output=True, text=True, env=env)
logger.info(" %s solved in %.0fs", platform, time.monotonic() - started)

# An unsatisfiable set fails here, with the solver's own explanation
if result.returncode != 0:
raise RuntimeError(f"the solve failed:\n{result.stdout}{result.stderr}")

# LINK lists every package the environment would contain, dependencies
# included, each with the version the solver chose
linked = json.loads(result.stdout)["actions"]["LINK"]
return {package["name"]: package["version"] for package in linked}


def apply(source: str, solved: dict[str, str]) -> tuple[str, list[str]]:
"""
Write the solved versions into the exact pins of the source.

Returns the new source and one ``name old -> new`` line per change.
Raises ValueError for a pinned package the solution does not contain.
"""
lines, changes = [], []
for line in source.splitlines():
# Only exact pins are rewritten; every other line is copied as it is
match = EXACT.match(line)
if match is not None:
name, old = match["name"], match["version"]

# Every pinned package is a direct request, so a solution without
# it means the solve and the file disagree
if name not in solved:
raise ValueError(f"{name} is pinned but not in the solution")

# Replace the version alone, keeping any trailing comment
if solved[name] != old:
line = f" - {name}={solved[name]}{match['rest']}"
changes.append(f"{name} {old} -> {solved[name]}")
lines.append(line)
return "\n".join(lines) + "\n", changes


def tighten(
solutions: dict[str, dict[str, str]], current: dict[str, str]
) -> set[tuple[str, str]]:
"""
Return the upper bounds that bring the platforms towards one solution.

Each bound is a ``(name, bound)`` pair. An empty result means no pinned
package is on a new pre-release and every platform chose the same
versions.
"""
# A pinned package the solver moved to a pre-release, on any platform, is
# capped just below it. A pin that is already a pre-release is left alone,
# as a cap at its own version could never be met.
snapshots = set()
for platform, solution in solutions.items():
for name, version in current.items():
chosen = solution.get(name, version)
if chosen != version and PRERELEASE.search(chosen):
logger.info(
" %s: %s chose pre-release %s, capping below it",
name,
platform,
chosen,
)
snapshots.add((name, f"<{chosen}"))
if snapshots:
return snapshots

# A pinned package the platforms disagree on is capped at every version
# chosen, which holds each platform to the lowest of them. A package a
# platform does not install is judged on the platforms that do.
caps = set()
for name in current:
versions = {p: s[name] for p, s in solutions.items() if name in s}
if len(set(versions.values())) > 1:
logger.info(
" %s: platforms disagree (%s), capping at the lowest",
name,
", ".join(f"{p} {v}" for p, v in versions.items()),
)
caps |= {(name, f"<={version}") for version in versions.values()}
return caps


def main() -> None:
argparse.ArgumentParser(description=__doc__.splitlines()[0]).parse_args()

# Progress goes to stderr, leaving stdout to the changes alone
logging.basicConfig(level=logging.INFO, format="%(message)s")

source = SOURCE.read_text()
current = pinned(source)
bounds: dict[str, set[str]] = {}
rounds = 0
try:
while True:
rounds += 1
caps = sum(len(b) for b in bounds.values())
logger.info(
"Round %d: solving %d platforms, %d caps", rounds, len(PLATFORMS), caps
)
# Solve every platform from the same loosened file
loosened = loosen(source, bounds)
solutions = {
platform: solve(loosened, platform) for platform in PLATFORMS
}

# Stop once the platforms agree. Every new bound excludes a version
# some platform just chose, so each pass makes progress and the
# loop ends; a bound already in place would mean it cannot.
tighter = tighten(solutions, current)
if not tighter:
break
if all(bound in bounds.get(name, ()) for name, bound in tighter):
raise RuntimeError("the platforms cannot agree on one set of pins")
for name, bound in tighter:
bounds.setdefault(name, set()).add(bound)

# The platforms agree, so any one of them gives each pinned version
solved: dict[str, str] = {}
for solution in solutions.values():
solved |= solution
updated, changes = apply(source, solved)
logger.info("Agreed after %d rounds, %d pins changed", rounds, len(changes))
except (RuntimeError, ValueError) as error:
sys.exit(f"{SOURCE}: {error}")

# The file is only touched once every step has succeeded, and the
# changes go to stdout for the pull request body
SOURCE.write_text(updated)
print("\n".join(changes))


if __name__ == "__main__":
main()
21 changes: 13 additions & 8 deletions .github/workflows/build_test.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
on: [push, pull_request]
on:
push:
branches: ["main"]
pull_request: null
workflow_dispatch: null
permissions:
contents: read
jobs:
build_test:
name: Build/Test
Expand All @@ -14,27 +20,26 @@ jobs:
# fresh conda install may not preserve, so compare contents instead.
CCACHE_COMPILERCHECK: content
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# The runner context is not available in the job env, so the cache
# directories are set here for every later step.
- name: Set cache directories
run: |
echo "CCACHE_DIR=${RUNNER_TEMP}/ccache" >> "$GITHUB_ENV"
echo "DEPS_DIR=${RUNNER_TEMP}/deps" >> "$GITHUB_ENV"
- uses: conda-incubator/setup-miniconda@v4
- uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0
with:
channels: conda-forge
conda-remove-defaults: true
- name: Install dependencies
run: |
conda env update --file requirements.txt --name test
conda install --yes --name test ccache
conda env update --file environment.yml --name test
conda list
# Each run restores the newest cache under the prefix and saves its
# own under the commit, so the cache rolls forward rather than
# freezing at the first save.
- name: Restore ccache
uses: actions/cache@v6
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ env.CCACHE_DIR }}
key: ccache-${{ runner.os }}-${{ github.sha }}
Expand All @@ -45,10 +50,10 @@ jobs:
# so the key covers the file that pins them and the one that chooses
# the toolchain.
- name: Restore dependency sources
uses: actions/cache@v6
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ env.DEPS_DIR }}
key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }}
key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'environment.yml') }}
# List each restored dependency with the commit it sits at. The
# directory starts empty on every job, so anything in it came from
# the cache. A login shell runs ~/.bash_logout on exit, which can fail
Expand Down
Loading
Loading