Fetch the sysimage on import and make it portable

README.md

@@ -14,6 +14,13 @@ The source code for `galley-jl-python` is available on GitHub at [https://github
 pip install galley-jl-python
 ```
 
+Galley needs a Python linked against **OpenSSL 3.5 or newer**. It runs Julia
+1.12, which bundles OpenSSL 3.5 in the same process as Python. Pythons from
+conda-forge, including through pixi or conda, meet this. Many system and
+pyenv-built Pythons still use OpenSSL 3.0, and there `import galley_jl_python`
+fails with an error that says so. Check with
+`python -c "import ssl; print(ssl.OPENSSL_VERSION)"`.
+
 ## Contributing
 
 ### Development environment
@@ -58,16 +65,19 @@ Options:
 ### Julia sysimage
 
 Most of Galley's startup time is Julia compiling Finch itself. A prebuilt Julia
-sysimage removes it: the first operations of a session drop from minutes to a
-few seconds.
-
-- `pixi run fetch-sysimage` downloads the image for your platform into
-  `~/.cache/galley-jl-python/` (`GALLEY_JL_PYTHON_CACHE` overrides this).
-  `pixi run compile` and `pixi run test` run this step first.
-- `pixi run build-sysimage` builds the image locally instead. This takes hours.
-- `import galley_jl_python` loads a cached image automatically when it matches
-  the current Julia environment. Otherwise Julia starts without it. Set
-  `GALLEY_JL_PYTHON_SYSIMAGE=0` to turn this off.
+sysimage removes it: the first matmul of a session drops from about 4.5 minutes
+to a few seconds.
+
+- `import galley_jl_python` uses the image for the current platform and Julia
+  environment. If it isn't cached in `~/.cache/galley-jl-python/`
+  (`GALLEY_JL_PYTHON_CACHE` overrides this), the import downloads it from the
+  project's GitHub releases first, about 900 MB once per environment. It then
+  checks that the image loads, and if there's no image or anything fails, Julia
+  starts without it. Set `GALLEY_JL_PYTHON_SYSIMAGE=0` to turn all of this off.
+- `pixi run fetch-sysimage` downloads the image ahead of time. `pixi run compile`
+  and `pixi run test` run this step first.
+- `pixi run build-sysimage` builds the image locally instead, in about half an
+  hour.
 
 An image works only with the exact Julia and package versions it was built
 from. For this reason `src/galley_jl_python/juliapkg.json` pins Julia and every
@@ -82,7 +92,7 @@ To update the Julia dependencies:
 
 Pushing the new pins to `main` runs the "Sysimage" GitHub Action. It builds
 images for Linux, macOS and Windows and publishes them to a `sysimage-<hash>`
-GitHub release, where `fetch-sysimage` finds them. The action can also be run
+GitHub release, where imports and `fetch-sysimage` find them. The action can also be run
 manually from the Actions tab.
 
 ### Publishing

scripts/sysimage/build_sysimage.jl

@@ -13,13 +13,17 @@ if Base.find_package("PackageCompiler") === nothing
 end
 using PackageCompiler
 
+# PythonCall must be in the image: loading it on top of the image invalidates
+# much of the image's code (~40 s of recompiling). relocate_pythoncall.jl stops
+# it from remembering the build machine's install path.
 packages = ["Finch", "HDF5", "NPZ", "TensorMarket", "PythonCall"]
 
 create_sysimage(
     packages;
     project=project,
     sysimage_path=output,
     precompile_statements_file=statements_file,
+    script=joinpath(@__DIR__, "relocate_pythoncall.jl"),
     # a generic target so the image runs on any x86_64 CPU, not just this one
     cpu_target=PackageCompiler.default_app_cpu_target(),
     include_transitive_dependencies=true,

scripts/sysimage/fetch.py

@@ -1,14 +1,11 @@
 """Download the prebuilt sysimage for this platform and Julia environment.
 
+Importing galley does this automatically; this script does it ahead of time.
 CI publishes one image per platform to the GitHub release tagged
 ``sysimage-<key>``, where the key identifies the exact Julia environment.
 """
 
-import os
 import sys
-import tempfile
-import urllib.error
-import urllib.request
 from pathlib import Path
 
 HERE = Path(__file__).resolve().parent
@@ -19,39 +16,15 @@
 
 def main() -> None:
     key = sysimage.current_key()
-    image = sysimage.image_path(key)
-    if image.is_file():
-        print(f"[sysimage] already present: {image}")
-        return
-
-    url = f"{sysimage.RELEASES_URL}/{sysimage.release_tag(key)}/{image.name}"
-    print(f"[sysimage] downloading {url}", flush=True)
-    image.parent.mkdir(parents=True, exist_ok=True)
-    fd, partial = tempfile.mkstemp(dir=image.parent, suffix=".part")
-    try:
-        with os.fdopen(fd, "wb") as out, urllib.request.urlopen(url) as resp:
-            while chunk := resp.read(1 << 20):
-                out.write(chunk)
-        os.replace(partial, image)
-    # never fail: other tasks depend on this one, and galley works without an
-    # image, just with a slower first call
-    except urllib.error.HTTPError as e:
-        if e.code != 404:
-            print(f"[sysimage] download failed ({e}); continuing without it.")
-            return
+    image = sysimage.fetch(key)
+    if image is not None:
+        print(f"[sysimage] {image}")
+    else:
         print(
-            f"[sysimage] no prebuilt image for this platform and Julia environment "
-            f"({image.name}); continuing without it. Build one locally with "
-            "`pixi run build-sysimage`."
+            f"[sysimage] no prebuilt image available for {sysimage.image_name(key)}"
+            " (or another process is downloading it); galley works without it. "
+            "Build one locally with `pixi run build-sysimage`."
         )
-        return
-    except (urllib.error.URLError, OSError) as e:
-        print(f"[sysimage] download failed ({e}); continuing without it.")
-        return
-    finally:
-        if os.path.exists(partial):
-            os.remove(partial)
-    print(f"[sysimage] saved {image}")
 
 
 if __name__ == "__main__":

scripts/sysimage/relocate_pythoncall.jl

@@ -0,0 +1,83 @@
+# Run by PackageCompiler in the process that writes the sysimage (the `script`
+# argument), so this redefinition is what the image contains.
+#
+# PythonCall's `init_stdlib` finds juliacall's `init.jl` through
+# `const ROOT_DIR = dirname(dirname(@__DIR__))`. In a sysimage that constant is
+# frozen to the build machine's path, so juliacall fails to start anywhere else.
+# This copy of the function finds `init.jl` in the Python `juliacall` package
+# instead, which is already in `sys.modules` when Python embeds Julia and holds
+# an identical copy of the file. It is otherwise PythonCall 0.9.36's function.
+#
+# Leaving PythonCall out of the image avoids this but costs ~40 s on the first
+# call: loading it on top of the image invalidates much of the image's code.
+using PythonCall
+
+pkgversion(PythonCall) == v"0.9.36" || error(
+    "PythonCall is $(pkgversion(PythonCall)), not 0.9.36: compare its " *
+    "src/Core/stdlib.jl with scripts/sysimage/relocate_pythoncall.jl and update it",
+)
+
+@eval PythonCall.Core function init_stdlib()
+
+    # check word size
+    pywordsize = pygt(Bool, pysysmodule.maxsize, Int64(2)^32) ? 64 : 32
+    pywordsize == Sys.WORD_SIZE ||
+        error("Julia is $(Sys.WORD_SIZE)-bit but Python is $(pywordsize)-bit")
+
+    if C.CTX.is_embedded
+
+        # This uses some internals, but Base._start() gets the state more like Julia
+        # is if you call the executable directly, in particular it creates workers when
+        # the --procs argument is given.
+        juliacall_dir = pystr_asstring(
+            pyosmodule.path.dirname(pysysmodule.modules["juliacall"].__file__),
+        )
+        push!(Base.Core.ARGS, joinpath(juliacall_dir, "init.jl"))
+        Base._start()
+        Base.eval(:(PROGRAM_FILE = ""))
+
+        # if Python is interactive, ensure Julia is too
+        if pyhasattr(pysysmodule, "ps1")
+            Base.eval(:(is_interactive = true))
+            Base.load_InteractiveUtils()
+        end
+
+    else
+
+        # set sys.argv
+        pysysmodule.argv = pylist([""; ARGS])
+
+        # some modules test for interactivity by checking if sys.ps1 exists
+        if isinteractive() && !pyhasattr(pysysmodule, "ps1")
+            pysysmodule.ps1 = ">>> "
+        end
+
+        # add hook to perform certain actions when certain modules are loaded
+        g = pydict()
+        pyexec(
+            """
+     import sys
+     class JuliaCompatHooks:
+         def __init__(self):
+             self.hooks = {}
+         def find_spec(self, name, path=None, target=None):
+             hs = self.hooks.get(name)
+             if hs is not None:
+                 for h in hs:
+                     h()
+         def add_hook(self, name, h):
+             if name not in self.hooks:
+                 self.hooks[name] = [h]
+             else:
+                 self.hooks[name].append(h)
+             if name in sys.modules:
+                 h()
+     JULIA_COMPAT_HOOKS = JuliaCompatHooks()
+     sys.meta_path.insert(0, JULIA_COMPAT_HOOKS)
+     """,
+            g,
+        )
+        pycopy!(pymodulehooks, g["JULIA_COMPAT_HOOKS"])
+    end
+
+end