From eb074a7f42a61b4a5094104b4cdf0bd3ad1d0bcf Mon Sep 17 00:00:00 2001 From: kylebd99 Date: Sun, 27 Sep 2026 19:21:46 -0400 Subject: [PATCH 1/3] Fetch the sysimage on import and make it portable Published images only worked on the machine that built them: PythonCall stores its install path in `const ROOT_DIR`, and juliacall reads it at startup, so on any other machine Julia failed with `SystemError: opening file "/home/runner/.julia/packages/PythonCall/...`. - Leave PythonCall out of the image, so it loads from the user's depot. - Add a format version to image names (`galley-v2-...`) so the fixed images don't collide with the broken ones already published. - On import, download the image for this environment if it isn't cached, and use it right away. One process downloads at a time; a 404 is remembered for a day; interrupted downloads resume; and each image is started once in a subprocess before it's used, so a bad image can never break `import galley_jl_python`. GALLEY_JL_PYTHON_SYSIMAGE=0 turns it off. Co-Authored-By: Claude Opus 5.5 --- README.md | 25 +++-- scripts/sysimage/build_sysimage.jl | 5 +- scripts/sysimage/fetch.py | 43 ++------ src/galley_jl_python/_sysimage.py | 161 +++++++++++++++++++++++++++-- 4 files changed, 178 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 90e80e5..44124ae 100644 --- a/README.md +++ b/README.md @@ -58,16 +58,19 @@ Options: ### Julia sysimage Most of Galley's startup time is Julia compiling Finch itself. A prebuilt Julia -sysimage removes it: the first operations of a session drop from minutes to a -few seconds. - -- `pixi run fetch-sysimage` downloads the image for your platform into - `~/.cache/galley-jl-python/` (`GALLEY_JL_PYTHON_CACHE` overrides this). - `pixi run compile` and `pixi run test` run this step first. -- `pixi run build-sysimage` builds the image locally instead. This takes hours. -- `import galley_jl_python` loads a cached image automatically when it matches - the current Julia environment. Otherwise Julia starts without it. Set - `GALLEY_JL_PYTHON_SYSIMAGE=0` to turn this off. +sysimage removes most of it: the first matmul of a session drops from about 4.5 +minutes to under a minute. + +- `import galley_jl_python` uses the image for the current platform and Julia + environment. If it isn't cached in `~/.cache/galley-jl-python/` + (`GALLEY_JL_PYTHON_CACHE` overrides this), the import downloads it from the + project's GitHub releases first, about 900 MB once per environment. It then + checks that the image loads, and if there's no image or anything fails, Julia + starts without it. Set `GALLEY_JL_PYTHON_SYSIMAGE=0` to turn all of this off. +- `pixi run fetch-sysimage` downloads the image ahead of time. `pixi run compile` + and `pixi run test` run this step first. +- `pixi run build-sysimage` builds the image locally instead, in about half an + hour. An image works only with the exact Julia and package versions it was built from. For this reason `src/galley_jl_python/juliapkg.json` pins Julia and every @@ -82,7 +85,7 @@ To update the Julia dependencies: Pushing the new pins to `main` runs the "Sysimage" GitHub Action. It builds images for Linux, macOS and Windows and publishes them to a `sysimage-` -GitHub release, where `fetch-sysimage` finds them. The action can also be run +GitHub release, where imports and `fetch-sysimage` find them. The action can also be run manually from the Actions tab. ### Publishing diff --git a/scripts/sysimage/build_sysimage.jl b/scripts/sysimage/build_sysimage.jl index fcd8b59..6f17114 100644 --- a/scripts/sysimage/build_sysimage.jl +++ b/scripts/sysimage/build_sysimage.jl @@ -13,7 +13,10 @@ if Base.find_package("PackageCompiler") === nothing end using PackageCompiler -packages = ["Finch", "HDF5", "NPZ", "TensorMarket", "PythonCall"] +# PythonCall stays out: it stores its own install path in a `const ROOT_DIR`, +# which juliacall reads at startup, so an image containing it only works on the +# machine that built it. Loaded from the user's depot, it finds its own files. +packages = ["Finch", "HDF5", "NPZ", "TensorMarket"] create_sysimage( packages; diff --git a/scripts/sysimage/fetch.py b/scripts/sysimage/fetch.py index 725569e..cd3c994 100644 --- a/scripts/sysimage/fetch.py +++ b/scripts/sysimage/fetch.py @@ -1,14 +1,11 @@ """Download the prebuilt sysimage for this platform and Julia environment. +Importing galley does this automatically; this script does it ahead of time. CI publishes one image per platform to the GitHub release tagged ``sysimage-``, where the key identifies the exact Julia environment. """ -import os import sys -import tempfile -import urllib.error -import urllib.request from pathlib import Path HERE = Path(__file__).resolve().parent @@ -19,39 +16,15 @@ def main() -> None: key = sysimage.current_key() - image = sysimage.image_path(key) - if image.is_file(): - print(f"[sysimage] already present: {image}") - return - - url = f"{sysimage.RELEASES_URL}/{sysimage.release_tag(key)}/{image.name}" - print(f"[sysimage] downloading {url}", flush=True) - image.parent.mkdir(parents=True, exist_ok=True) - fd, partial = tempfile.mkstemp(dir=image.parent, suffix=".part") - try: - with os.fdopen(fd, "wb") as out, urllib.request.urlopen(url) as resp: - while chunk := resp.read(1 << 20): - out.write(chunk) - os.replace(partial, image) - # never fail: other tasks depend on this one, and galley works without an - # image, just with a slower first call - except urllib.error.HTTPError as e: - if e.code != 404: - print(f"[sysimage] download failed ({e}); continuing without it.") - return + image = sysimage.fetch(key) + if image is not None: + print(f"[sysimage] {image}") + else: print( - f"[sysimage] no prebuilt image for this platform and Julia environment " - f"({image.name}); continuing without it. Build one locally with " - "`pixi run build-sysimage`." + f"[sysimage] no prebuilt image available for {sysimage.image_name(key)}" + " (or another process is downloading it); galley works without it. " + "Build one locally with `pixi run build-sysimage`." ) - return - except (urllib.error.URLError, OSError) as e: - print(f"[sysimage] download failed ({e}); continuing without it.") - return - finally: - if os.path.exists(partial): - os.remove(partial) - print(f"[sysimage] saved {image}") if __name__ == "__main__": diff --git a/src/galley_jl_python/_sysimage.py b/src/galley_jl_python/_sysimage.py index a96ffbe..20b4bdf 100644 --- a/src/galley_jl_python/_sysimage.py +++ b/src/galley_jl_python/_sysimage.py @@ -1,4 +1,4 @@ -"""Locate a prebuilt Julia sysimage and point juliacall at it. +"""Locate, and if needed download, a prebuilt Julia sysimage for juliacall. This module must not import juliacall: it runs before Julia starts. @@ -7,22 +7,36 @@ lives in a per-user cache directory. An image is used only when its name matches the current environment; otherwise Julia starts normally. -Get an image with ``pixi run fetch-sysimage`` (downloads the one CI built for -this environment) or ``pixi run build-sysimage`` (builds it locally). Set -``GALLEY_JL_PYTHON_SYSIMAGE=0`` to never load one, or -``PYTHON_JULIACALL_SYSIMAGE`` to use a specific image. +On import, galley downloads the image CI published for this platform and +environment if it isn't cached yet, and uses it right away. Set +``GALLEY_JL_PYTHON_SYSIMAGE=0`` to never download or load one, or +``PYTHON_JULIACALL_SYSIMAGE`` to use a specific image. In a galley checkout, +``pixi run build-sysimage`` builds one locally. """ import hashlib import json import os import platform +import subprocess import sys +import tempfile +import time import tomllib +import urllib.error +import urllib.request from pathlib import Path RELEASES_URL = "https://github.com/finch-tensor/galley-jl-python/releases/download" +# Bump when images built from the same environment become incompatible, so the +# new images get new names instead of reusing the published ones. +# 2: PythonCall no longer in the image (v1 images only worked on the build host) +IMAGE_FORMAT = 2 _EXTENSIONS = {"linux": "so", "darwin": "dylib", "win32": "dll"} +# after a 404, don't ask again for this long (images are published after CI) +_MISSING_TTL = 24 * 60 * 60 +# a lock older than this is left over from a download that was killed +_STALE_LOCK = 2 * 60 * 60 def cache_dir() -> Path: @@ -55,7 +69,8 @@ def release_tag(key: str) -> str: def image_name(key: str) -> str: ext = _EXTENSIONS.get(sys.platform, "so") - return f"galley-{sys.platform}-{platform.machine().lower()}-{key}.{ext}" + arch = platform.machine().lower() + return f"galley-v{IMAGE_FORMAT}-{sys.platform}-{arch}-{key}.{ext}" def image_path(key: str) -> Path: @@ -68,12 +83,140 @@ def current_key() -> str: return env_key(juliapkg.project()) +def _log(message: str) -> None: + print(f"[galley-jl-python] {message}", file=sys.stderr, flush=True) + + +def _acquire_lock(lock: Path) -> bool: + for _ in range(2): + try: + os.close(os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)) + return True + except FileExistsError: + try: + if time.time() - lock.stat().st_mtime < _STALE_LOCK: + return False + lock.unlink() + except FileNotFoundError: + pass + return False + + +def _download(url: str, dest: Path, attempts: int = 4) -> None: + """Download ``url`` to ``dest``, resuming after a dropped connection.""" + size = done = 0 + next_report = 0.25 + for attempt in range(attempts): + headers = {"Range": f"bytes={done}-"} if done else {} + try: + req = urllib.request.Request(url, headers=headers) + with ( + urllib.request.urlopen(req, timeout=60) as resp, + dest.open("ab") as out, + ): + if done and resp.status != 206: # server ignored the range + out.truncate(0) + done = 0 + if not size: + size = done + int(resp.headers.get("Content-Length") or 0) + _log( + f"downloading the prebuilt Julia sysimage ({size >> 20} MB), " + "once per environment; set GALLEY_JL_PYTHON_SYSIMAGE=0 to skip" + ) + while chunk := resp.read(1 << 20): + out.write(chunk) + done += len(chunk) + if size and done / size >= next_report: + _log(f" {done * 100 // size}%") + next_report += 0.25 + if not size or done >= size: + return + except urllib.error.HTTPError: + raise + except (urllib.error.URLError, OSError): + if attempt == attempts - 1: + raise + _log( + f" connection dropped at {done >> 20} MB; resuming" + if done + else " couldn't connect; retrying" + ) + + +def _works(image: Path) -> bool: + """Check that juliacall starts with ``image``, in a throwaway process.""" + env = {**os.environ, "PYTHON_JULIACALL_SYSIMAGE": str(image)} + env["GALLEY_JL_PYTHON_SYSIMAGE"] = "0" + try: + result = subprocess.run( + [sys.executable, "-c", "from juliacall import Main; Main.seval('1 + 1')"], + env=env, + capture_output=True, + timeout=600, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return result.returncode == 0 + + +def fetch(key: str) -> Path | None: + """Download the published image for ``key`` into the cache. + + Returns the image path, or None if there is no image or it couldn't be + downloaded. Never raises: galley works without an image, just with a slow + first call. If another process is already downloading, returns None + immediately instead of waiting. + """ + image = image_path(key) + if image.is_file(): + return image + missing = image.with_name(image.name + ".missing") + lock = image.with_name(image.name + ".lock") + try: + if missing.is_file() and time.time() - missing.stat().st_mtime < _MISSING_TTL: + return None + image.parent.mkdir(parents=True, exist_ok=True) + if not _acquire_lock(lock): + return None + except OSError: + return None + + url = f"{RELEASES_URL}/{release_tag(key)}/{image.name}" + partial = None + try: + fd, partial = tempfile.mkstemp(dir=image.parent, suffix=".part") + os.close(fd) + _download(url, Path(partial)) + if not _works(Path(partial)): + _log("the downloaded sysimage doesn't load here; starting without it") + missing.touch() + return None + os.chmod(partial, 0o644) + os.replace(partial, image) + partial = None + _log(f"saved {image}") + return image + except urllib.error.HTTPError as e: + if e.code == 404: + missing.touch() + else: + _log(f"sysimage download failed ({e}); starting without it") + return None + except (urllib.error.URLError, OSError) as e: + _log(f"sysimage download failed ({e}); starting without it") + return None + finally: + if partial is not None and os.path.exists(partial): + os.remove(partial) + lock.unlink(missing_ok=True) + + def configure() -> None: - """Point juliacall at the cached sysimage for this environment, if any.""" + """Point juliacall at this environment's sysimage, downloading it if needed.""" if os.environ.get("GALLEY_JL_PYTHON_SYSIMAGE") == "0": return if "PYTHON_JULIACALL_SYSIMAGE" in os.environ: return - image = image_path(current_key()) - if image.is_file(): + image = fetch(current_key()) + if image is not None: os.environ["PYTHON_JULIACALL_SYSIMAGE"] = str(image) From 168261bdf77b24d0c935d8a963870c3450f42f74 Mon Sep 17 00:00:00 2001 From: kylebd99 Date: Sun, 27 Sep 2026 20:03:14 -0400 Subject: [PATCH 2/3] Keep PythonCall in the sysimage, relocated Leaving PythonCall out made images portable but slow: loading it on top of the image invalidated much of the image's code, and the first matmul spent 39 s recompiling it. PythonCall only uses its frozen install path, when Python embeds Julia, to locate juliacall's init.jl. relocate_pythoncall.jl, run by PackageCompiler while writing the image, redefines init_stdlib to find the identical copy in the Python juliacall package instead. The build refuses anything but PythonCall 0.9.36, so an upgrade forces the copied function to be checked again. In a separate project whose Julia depot differs from the build's, import takes 2.6-3.1 s and the first matmul 3.6-4.2 s, with no recompilation. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- scripts/sysimage/build_sysimage.jl | 9 +-- scripts/sysimage/relocate_pythoncall.jl | 83 +++++++++++++++++++++++++ src/galley_jl_python/_sysimage.py | 2 +- 4 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 scripts/sysimage/relocate_pythoncall.jl diff --git a/README.md b/README.md index 44124ae..3e444de 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,8 @@ Options: ### Julia sysimage Most of Galley's startup time is Julia compiling Finch itself. A prebuilt Julia -sysimage removes most of it: the first matmul of a session drops from about 4.5 -minutes to under a minute. +sysimage removes it: the first matmul of a session drops from about 4.5 minutes +to a few seconds. - `import galley_jl_python` uses the image for the current platform and Julia environment. If it isn't cached in `~/.cache/galley-jl-python/` diff --git a/scripts/sysimage/build_sysimage.jl b/scripts/sysimage/build_sysimage.jl index 6f17114..7318334 100644 --- a/scripts/sysimage/build_sysimage.jl +++ b/scripts/sysimage/build_sysimage.jl @@ -13,16 +13,17 @@ if Base.find_package("PackageCompiler") === nothing end using PackageCompiler -# PythonCall stays out: it stores its own install path in a `const ROOT_DIR`, -# which juliacall reads at startup, so an image containing it only works on the -# machine that built it. Loaded from the user's depot, it finds its own files. -packages = ["Finch", "HDF5", "NPZ", "TensorMarket"] +# PythonCall must be in the image: loading it on top of the image invalidates +# much of the image's code (~40 s of recompiling). relocate_pythoncall.jl stops +# it from remembering the build machine's install path. +packages = ["Finch", "HDF5", "NPZ", "TensorMarket", "PythonCall"] create_sysimage( packages; project=project, sysimage_path=output, precompile_statements_file=statements_file, + script=joinpath(@__DIR__, "relocate_pythoncall.jl"), # a generic target so the image runs on any x86_64 CPU, not just this one cpu_target=PackageCompiler.default_app_cpu_target(), include_transitive_dependencies=true, diff --git a/scripts/sysimage/relocate_pythoncall.jl b/scripts/sysimage/relocate_pythoncall.jl new file mode 100644 index 0000000..7f898ca --- /dev/null +++ b/scripts/sysimage/relocate_pythoncall.jl @@ -0,0 +1,83 @@ +# Run by PackageCompiler in the process that writes the sysimage (the `script` +# argument), so this redefinition is what the image contains. +# +# PythonCall's `init_stdlib` finds juliacall's `init.jl` through +# `const ROOT_DIR = dirname(dirname(@__DIR__))`. In a sysimage that constant is +# frozen to the build machine's path, so juliacall fails to start anywhere else. +# This copy of the function finds `init.jl` in the Python `juliacall` package +# instead, which is already in `sys.modules` when Python embeds Julia and holds +# an identical copy of the file. It is otherwise PythonCall 0.9.36's function. +# +# Leaving PythonCall out of the image avoids this but costs ~40 s on the first +# call: loading it on top of the image invalidates much of the image's code. +using PythonCall + +pkgversion(PythonCall) == v"0.9.36" || error( + "PythonCall is $(pkgversion(PythonCall)), not 0.9.36: compare its " * + "src/Core/stdlib.jl with scripts/sysimage/relocate_pythoncall.jl and update it", +) + +@eval PythonCall.Core function init_stdlib() + + # check word size + pywordsize = pygt(Bool, pysysmodule.maxsize, Int64(2)^32) ? 64 : 32 + pywordsize == Sys.WORD_SIZE || + error("Julia is $(Sys.WORD_SIZE)-bit but Python is $(pywordsize)-bit") + + if C.CTX.is_embedded + + # This uses some internals, but Base._start() gets the state more like Julia + # is if you call the executable directly, in particular it creates workers when + # the --procs argument is given. + juliacall_dir = pystr_asstring( + pyosmodule.path.dirname(pysysmodule.modules["juliacall"].__file__), + ) + push!(Base.Core.ARGS, joinpath(juliacall_dir, "init.jl")) + Base._start() + Base.eval(:(PROGRAM_FILE = "")) + + # if Python is interactive, ensure Julia is too + if pyhasattr(pysysmodule, "ps1") + Base.eval(:(is_interactive = true)) + Base.load_InteractiveUtils() + end + + else + + # set sys.argv + pysysmodule.argv = pylist([""; ARGS]) + + # some modules test for interactivity by checking if sys.ps1 exists + if isinteractive() && !pyhasattr(pysysmodule, "ps1") + pysysmodule.ps1 = ">>> " + end + + # add hook to perform certain actions when certain modules are loaded + g = pydict() + pyexec( + """ + import sys + class JuliaCompatHooks: + def __init__(self): + self.hooks = {} + def find_spec(self, name, path=None, target=None): + hs = self.hooks.get(name) + if hs is not None: + for h in hs: + h() + def add_hook(self, name, h): + if name not in self.hooks: + self.hooks[name] = [h] + else: + self.hooks[name].append(h) + if name in sys.modules: + h() + JULIA_COMPAT_HOOKS = JuliaCompatHooks() + sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) + """, + g, + ) + pycopy!(pymodulehooks, g["JULIA_COMPAT_HOOKS"]) + end + +end diff --git a/src/galley_jl_python/_sysimage.py b/src/galley_jl_python/_sysimage.py index 20b4bdf..9764460 100644 --- a/src/galley_jl_python/_sysimage.py +++ b/src/galley_jl_python/_sysimage.py @@ -30,7 +30,7 @@ RELEASES_URL = "https://github.com/finch-tensor/galley-jl-python/releases/download" # Bump when images built from the same environment become incompatible, so the # new images get new names instead of reusing the published ones. -# 2: PythonCall no longer in the image (v1 images only worked on the build host) +# 2: PythonCall finds juliacall from Python (v1 only worked on the build host) IMAGE_FORMAT = 2 _EXTENSIONS = {"linux": "so", "darwin": "dylib", "win32": "dll"} # after a 404, don't ask again for this long (images are published after CI) From f2919404884b6a0efcb9c925b31dce8fc881a171 Mon Sep 17 00:00:00 2001 From: kylebd99 Date: Sun, 27 Sep 2026 20:11:18 -0400 Subject: [PATCH 3/3] Require a Python with OpenSSL 3.5 or newer galley pins Julia 1.12, which bundles OpenSSL 3.5 in the same process as Python, so juliapkg only allows it with a Python linked against OpenSSL 3.5+. With older OpenSSL (common: Ubuntu 24.04 and pyenv builds use 3.0), juliapkg failed with no compatible Julia version. Fail at import with an error that names the cause and the fix instead, and document the requirement. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 +++++++ src/galley_jl_python/julia.py | 19 +++++++++++++++++-- tests/test_openssl.py | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 tests/test_openssl.py diff --git a/README.md b/README.md index 3e444de..bee8101 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,13 @@ The source code for `galley-jl-python` is available on GitHub at [https://github pip install galley-jl-python ``` +Galley needs a Python linked against **OpenSSL 3.5 or newer**. It runs Julia +1.12, which bundles OpenSSL 3.5 in the same process as Python. Pythons from +conda-forge, including through pixi or conda, meet this. Many system and +pyenv-built Pythons still use OpenSSL 3.0, and there `import galley_jl_python` +fails with an error that says so. Check with +`python -c "import ssl; print(ssl.OPENSSL_VERSION)"`. + ## Contributing ### Development environment diff --git a/src/galley_jl_python/julia.py b/src/galley_jl_python/julia.py index 7ac7934..39f3c20 100644 --- a/src/galley_jl_python/julia.py +++ b/src/galley_jl_python/julia.py @@ -1,6 +1,21 @@ -import juliapkg # noqa: I001, F401 +import ssl # noqa: I001 -from . import _sysimage +# Julia 1.12, which galley pins, bundles OpenSSL 3.5, and Python and Julia share +# one process, so juliapkg only allows Julia 1.12 with a Python linked against +# OpenSSL 3.5 or newer. Without this check the failure is juliapkg finding no +# compatible Julia version. +if ssl.OPENSSL_VERSION_INFO[:2] < (3, 5): + raise ImportError( + "galley-jl-python needs a Python linked against OpenSSL 3.5 or newer, but " + f"this one uses {ssl.OPENSSL_VERSION}. Galley runs Julia 1.12, which " + "bundles OpenSSL 3.5 and shares this process with Python. Use a Python " + "from conda-forge, e.g. through pixi or conda, which ships a current " + "OpenSSL." + ) + +import juliapkg # noqa: E402, F401 + +from . import _sysimage # noqa: E402 _sysimage.configure() diff --git a/tests/test_openssl.py b/tests/test_openssl.py new file mode 100644 index 0000000..0cc2e7f --- /dev/null +++ b/tests/test_openssl.py @@ -0,0 +1,18 @@ +import subprocess +import sys + + +def test_old_openssl_gets_a_clear_error(): + # the check runs before Julia starts, so this doesn't initialize Julia + code = ( + "import ssl\n" + "ssl.OPENSSL_VERSION_INFO = (3, 0, 13, 0, 0)\n" + "ssl.OPENSSL_VERSION = 'OpenSSL 3.0.13 30 Jan 2024'\n" + "import galley_jl_python\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, timeout=120 + ) + assert result.returncode != 0 + assert "OpenSSL 3.5 or newer" in result.stderr + assert "OpenSSL 3.0.13" in result.stderr