Introducing Menudet: a little language for portable computation on arrays and tables

.github/workflows/build.yml

@@ -108,6 +108,17 @@ jobs:
         PYTEST_DEADMAN_SECONDS: "300"
         PYTEST_DEADMAN_SESSION_SECONDS: "1500"
 
+    - name: Certify bounded Menudet math against independent references
+      run: python scripts/certify_menudet_math.py --report menudet-math.json
+
+    - name: Upload Menudet math certification
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: menudet-math-${{ matrix.os }}-py${{ matrix.python-version }}-np${{ matrix.numpy-version || 'default' }}
+        path: menudet-math.json
+        if-no-files-found: ignore
+
     # TUIs are timing-sensitive and only boot a headless terminal: one OS is
     # enough, and a flake here is feedback, not a release blocker.
     - name: Test TUI (non-blocking)

.github/workflows/cibuildwheels.yml

@@ -1,6 +1,8 @@
 name: Python wheels
 
 on:
+  # Certify development revisions without creating a release tag or publishing.
+  workflow_dispatch:
   # Trigger the workflow only for tags and PRs to the main branch
   push:
     tags:
@@ -11,6 +13,20 @@ on:
 
 env:
   CIBW_BUILD_VERBOSITY: 1
+  # Every tested wheel checks portable descriptors/persistence and independent
+  # numeric references, not only whether the extension can be imported.
+  # Free-threaded CPython safely enables the GIL for this extension. Allow only
+  # that expected import warning; never force PYTHON_GIL=0 or claim no-GIL safety.
+  CIBW_TEST_COMMAND: >-
+    python -m pytest
+    {project}/tests/test_portable_artifact.py
+    {project}/tests/test_portable_descriptor.py
+    {project}/tests/test_portable_descriptor_host.py
+    {project}/tests/test_portable_lazy.py
+    {project}/tests/test_portable_policy.py
+    {project}/tests/test_portable_table_export.py
+    {project}/tests/test_dsl_portable.py -q -n 0 &&
+    python {project}/scripts/certify_menudet_math.py
   # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long
   # Now that github provides native arm64 runners, we can enable tests again
   # CIBW_TEST_SKIP: "*linux*aarch64*"
@@ -156,7 +172,9 @@ jobs:
       CIBW_BUILD: ${{ matrix.cibw_build }}
       # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths
       CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF"
-      CIBW_TEST_COMMAND: "pytest {project}/tests"
+      CIBW_TEST_COMMAND: >-
+        pytest {project}/tests &&
+        python {project}/scripts/certify_menudet_math.py
       # Pin the Pyodide version explicitly per target for reproducible builds.
       # Note: in-memory SChunk get_slice is broken on the Pyodide 0.29.x
       # Emscripten toolchain (works on 314); see gh-664. The affected test is
@@ -282,6 +300,9 @@ jobs:
           python -m pytest tests/test_open.py tests/test_vlmeta.py \
                            tests/ndarray/test_evaluate.py -q -p no:cacheprovider
 
+      - name: Certify Menudet on the installed abi3 wheel
+        run: python scripts/certify_menudet_math.py
+
 
   upload_pypi:
     needs: [ build_wheels, build_wheels_wasm, test_abi3_matrix ]

.github/workflows/wasm-tests.yml

@@ -19,7 +19,9 @@ jobs:
       # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths
       CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF"
       # cibuildwheel runs the suite automatically in the node/pyodide runtime
-      CIBW_TEST_COMMAND: "pytest {project}/tests"
+      CIBW_TEST_COMMAND: >-
+        pytest {project}/tests &&
+        python {project}/scripts/certify_menudet_math.py
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-python@v7

CMakeLists.txt

@@ -65,6 +65,7 @@ add_custom_command(
   COMMAND Python::Interpreter -m cython
           "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx" --output-file blosc2_ext.c
   DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx"
+          "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/dsl_artifact_bridge.h"
   VERBATIM)
 
 add_custom_command(
@@ -117,6 +118,7 @@ set(MINIEXPR_BUILD_SHARED OFF CACHE BOOL "Build miniexpr shared library" FORCE)
 set(MINIEXPR_BUILD_TESTS OFF CACHE BOOL "Build miniexpr tests" FORCE)
 set(MINIEXPR_BUILD_EXAMPLES OFF CACHE BOOL "Build miniexpr examples" FORCE)
 set(MINIEXPR_BUILD_BENCH OFF CACHE BOOL "Build miniexpr benchmarks" FORCE)
+option(MINIEXPR_BUILD_ARTIFACT "Build portable DSL artifact support" ON)
 # Keep miniexpr's and C-Blosc2's bundled files inside the blosc2 package.
 # Without this, their install rules use the default CMAKE_INSTALL_LIBDIR
 # ("lib"), which scikit-build places at site-packages/lib.
@@ -140,13 +142,20 @@ endif()
 
 FetchContent_Declare(miniexpr
     GIT_REPOSITORY https://github.com/Blosc/miniexpr.git
-    GIT_TAG 3f4db93a628aedc662b7d27a5f35c97ffaeb0424
+    GIT_TAG 3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4
     # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr
 )
 FetchContent_MakeAvailable(miniexpr)
 
 # Link against miniexpr static library
 target_link_libraries(blosc2_ext PRIVATE miniexpr_static)
+if(TARGET miniexpr_artifact)
+  target_link_libraries(blosc2_ext PRIVATE miniexpr_artifact)
+  target_compile_definitions(blosc2_ext PRIVATE B2_HAVE_PORTABLE_ARTIFACT=1)
+  FetchContent_GetProperties(yyjson SOURCE_DIR B2_YYJSON_SOURCE_DIR)
+  install(FILES "${B2_YYJSON_SOURCE_DIR}/LICENSE"
+    DESTINATION ${SKBUILD_METADATA_DIR}/licenses/miniexpr RENAME LICENSE-YYJSON)
+endif()
 if(APPLE)
   target_link_libraries(blosc2_ext PRIVATE "-framework Accelerate")
 endif()

RELEASE_NOTES.md

@@ -1,5 +1,49 @@
 # Release notes
 
+## Changes from 4.14.2 to 4.15.0 (in development)
+
+### Introducing the Menudet 1.0 draft
+
+**Menudet — a little language for portable computation on arrays and tables.**
+
+- Export normalized `DSLKernel` authoring as validated, standalone JSON artifacts
+  with typed signatures and exact captured constants. Native miniexpr executes
+  imports without reconstructing Python functions or falling back to Python.
+- Support checked integer arithmetic, operand-typed floating computation, control
+  flow, fixed-width strings, logical array coordinates and ordered block reductions.
+- Keep logical reduction groups stable through persistence, partial reads and
+  output rechunking. True block-scalar results expose the logical block-grid shape.
+- Validate kernel inputs to `CTable.add_computed_column()` and
+  `add_generated_column()` as Menudet artifacts with implicit independent-row
+  semantics, including fixed-shape row reductions. Named bindings are supported;
+  positional bindings and column-bound DSL LazyUDFs adopt the same semantics.
+  New cross-row computations must use array partitions or table aggregation,
+  rather than implicit execution blocks or append batches.
+- Remove the draft `add_portable_*` column methods and `row_domain` argument.
+  Fixed-shape row reductions use `cardinality="block_scalar"`.
+- New compliant DSLKernel-backed LazyUDF saves export the draft automatically;
+  unsupported saves fail validation before writing the destination.
+- Historical Python-specific recipes require explicit `deserialize="full"`.
+  MessagePack decoding is safe by default, including nested containers.
+- Saved legacy table-kernel recipes retain their old block/batch-dependent
+  behavior only with trusted, explicit `deserialize="full"`; safe-mode errors
+  explain the opt-in. Recipes are never automatically reinterpreted.
+- Ordinary LazyExpr persistence retains supported numerical calls, reductions,
+  methods and indexing under default loading, with existing expression validation.
+  Full-only nested payloads remain gated. String/lazy expressions and non-kernel
+  row transformers retain their existing table registration behavior.
+
+Menudet 1.0 is a **draft**, not a frozen compatibility promise. The existing
+`DSLKernel` and portable API names remain unchanged. This draft uses the typed
+interpreter; optional JIT requests fall back before execution, while required JIT
+rejects unsupported acceleration. Complex computation, callbacks, arbitrary
+memory access and non-strict floating-point modes are outside the portable contract.
+
+See the [Menudet reference](https://www.blosc.org/python-blosc2/reference/portable_dsl.html)
+for array/table examples, persistence policy and current limitations. Release
+certification is tracked separately; this development entry is not an announcement
+that 4.15.0 has been published.
+
 ## Changes from 4.14.1 to 4.14.2
 
 XXX version-specific blurb XXX

doc/conf.py

@@ -111,7 +111,14 @@ def genbody(f, func_list, lib="blosc2"):
     ],
 }
 
-exclude_patterns = ["_build", ".DS_Store", "**.ipynb_checkpoints", "tutorials/images/**"]
+exclude_patterns = [
+    "_build",
+    ".DS_Store",
+    "**.ipynb_checkpoints",
+    "tutorials/images/**",
+    "html/**",
+    "**/autofiles-stash/**",
+]
 
 html_show_sourcelink = False
 
@@ -306,7 +313,7 @@ def process_sig(app, what, name, obj, options, signature, return_annotation):
 # ``-W`` (as CI does) to turn that warning into a failure.
 
 _AUTODOC_DIRECTIVE = re.compile(
-    r"^\s*\.\.\s+(?:autoclass|autofunction|autodata|autoexception|autodecorator)::"
+    r"^\s*\.\.\s+(?:autoclass|autofunction|autodata|autoexception|autodecorator|py:class)::"
     r"\s*([\w.]+)"
 )
 _CURRENTMODULE = re.compile(r"^\s*\.\.\s+(?:currentmodule|module)::\s*([\w.]+)")

doc/getting_started/installation.rst

@@ -33,13 +33,13 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax:
      - The :doc:`b2view <../guides/b2view>` terminal browser (``textual``,
        ``textual-plotext``), including its in-terminal braille plot (the
        ``p`` key).  Required by the ``b2view`` command.
-    * - ``hires``
+   * - ``hires``
      - The high-resolution image view in b2view (the ``h`` key), which
        renders a real ``matplotlib`` image in the terminal
-        (``textual-image``, ``matplotlib``).  Includes ``tui``.
-    * - ``images``
-      - JPEG/PNG ordinary-file previews in b2view (``Pillow``, ``textual-image``).
-        Includes ``tui`` without requiring matplotlib. ``hires`` includes this extra.
+       (``textual-image``, ``matplotlib``).  Includes ``tui``.
+   * - ``images``
+     - JPEG/PNG ordinary-file previews in b2view (``Pillow``, ``textual-image``).
+       Includes ``tui`` without requiring matplotlib. ``hires`` includes this extra.
    * - ``parquet``
      - The ``parquet-to-blosc2`` converter (``pyarrow``); see
        :doc:`../guides/parquet_to_blosc2`.

doc/reference/dsl_syntax.md

@@ -3,6 +3,10 @@
 This is the practical reference for the DSL used by `@blosc2.dsl_kernel`
 functions (and checked by `blosc2.validate_dsl()`).
 It focuses on what works today and the most common gotchas.
+The portable subset is named **Menudet**, a little language for portable
+computation on arrays and tables. See the [Menudet reference](portable_dsl.rst)
+for the 1.0 draft contract. This page also documents ordinary full-DSL features;
+acceptance here does not imply portability.
 For usage walkthroughs and end-to-end examples, see the
 [LazyArray UDF DSL kernels tutorial](../tutorials/03.lazyarray-udf-kernels.ipynb).
 
@@ -18,6 +22,139 @@ def kernel(x, y):
 
 Use Python-style indentation and always return a value on the paths you execute.
 
+### Native source without a Python function
+
+`blosc2.DSLKernel.from_source(source)` constructs a native-only kernel without
+executing source as Python or inspecting Python globals:
+
+```python
+source = """# me:compiler=tcc
+def affine(x):
+    return x * 2.0 + 1.0
+"""
+kernel = blosc2.DSLKernel.from_source(source)
+result = blosc2.lazyudf(kernel, (x,), dtype="float64")[:]
+```
+
+The source and header pragmas are preserved verbatim. Input dtypes come from the
+supplied arrays; pass an explicit output dtype. The native compiler validates the
+body with these types during computation. The constructor inspects only the
+plain positional function header and does not certify portable-profile support.
+Frontend conveniences, captured globals, and Python-function fallback are not
+available on this path; use native function spellings such as `sin(x)`.
+
+This is separate from the existing Python-based persistence reconstruction path.
+Portable artifact export/import is still under development. The full native
+language reference and experimental portable profile live in miniexpr's
+`doc/dsl-syntax.md` and `doc/dsl-spec/1.0.md`, respectively. The portable profile
+and artifact contract are experimental drafts; see [portable artifacts](portable_dsl.rst)
+for the release boundary and legacy Python persistence policy.
+
+### Experimental portable-profile validation
+
+```python
+info = blosc2.validate_portable_dsl(source, {"x": "float64"}, "float64")
+if not info["valid"]:
+    print(info["status"], info["line"], info["column"], info["error"])
+```
+
+This native-only check uses raw source and explicit logical dtypes, with no
+frontend rewriting or captured globals. It neither executes the kernel nor
+invokes a JIT compiler. It reports version, signature, source, and unsupported
+feature errors separately. Potential runtime errors, including missing returns
+on some paths, do not automatically invalidate a program.
+
+Draft 1.0 admits Boolean, standard integer widths, float32/float64 and fixed strings,
+mixed operand-driven numeric typing, checked arithmetic, reductions and explicit ND context.
+It is separate from the ordinary full language described below.
+Validation is not a sandbox, backend-support probe, or a guarantee about
+arbitrary runtime data. Older native builds report `runtime_unsupported` rather
+than attempting Python validation. Native dependency integration for this
+experimental API is pending; development builds can use the local miniexpr source
+via CMake's `FETCHCONTENT_SOURCE_DIR_MINIEXPR` override.
+
+### Portable artifact export and import (experimental)
+
+Export with explicit input/output dtypes. Scalar globals and closure values are
+snapshotted into typed manifest constants, not inserted as Python literals:
+
+```python
+SCALE = 2.0
+
+
+@blosc2.dsl_kernel
+def scaled(x):
+    return x * SCALE
+
+
+artifact = scaled.export({"x": "float64"}, "float64")
+SCALE = 99.0  # Does not change the saved artifact.
+kernel = blosc2.PortableKernel.from_json(artifact, jit=False)
+values = kernel.evaluate({"x": np.arange(4, dtype=np.float64)})
+# values: array([0., 2., 4., 6.])
+```
+
+`export()` returns deterministic UTF-8 JSON text; save it with an ordinary file
+write. `from_json()` accepts text or bytes, not a pathname or decoded mapping.
+Import uses the same native JSON loader as C and does not need the author's module,
+function, decorators, globals, Python source reconstruction, or frontend rewrites.
+`evaluate()` returns a new NumPy array; `blosc2.asarray(values)` can materialize
+compressed storage. `kernel.lazy()` binds explicit logical partitions for lazy
+evaluation and portable persistence. Newly authored compliant DSLKernel-backed
+LazyUDF saves also normalize and validate portable native recipes automatically.
+
+Python `bool`, `int`, and `float` captures infer `bool`, `int64`, and `float64`.
+Supported NumPy scalar types retain their dtype. Use
+`capture_dtypes={"SCALE": "float32"}` to explicitly specialize a capture; finite
+float narrowing and explicit integer-to-float conversion round to the chosen dtype;
+out-of-range values reject. Arrays (including zero-dimensional arrays), objects, unsupported
+scalar types, unresolved names, and external callbacks cannot be captures.
+Export does not call the kernel or capture conversion hooks. It reuses supported
+authoring normalization, such as `np.sin(x)`, before native profile validation.
+Fixed byte/Unicode scalar snapshots are supported; arbitrary objects and callbacks are not.
+
+Native-only kernels can bind existing parameters as constants explicitly:
+
+```python
+author = blosc2.DSLKernel.from_source(
+    "# me:compiler=cc\ndef scaled(x, scale):\n    return x * scale\n"
+)
+artifact = author.export({"x": "float64"}, "float64", constants={"scale": 2.0})
+```
+
+Every source parameter requires exactly one runtime input or constant binding.
+Parameters and captures may have different supported dtypes; computation is
+operand-driven before output conversion. Manifest float constants preserve
+IEEE bit patterns, including signed zero and non-finite values; integer constants
+are range-checked decimal strings. Comments and retained native pragmas survive
+parameterization. Required strict FP semantics are independent of host defaults;
+compiler pragmas remain preferences. Draft 1.0 currently always uses the typed interpreter.
+
+Runtime arrays are bound by name in any mapping order and must have the exact
+declared logical dtype and common shape. The adapter converts endian order,
+alignment, and strides, but does not broadcast arrays or silently change their
+dtype. Constant-only kernels require `evaluate({}, shape=(...))`; empty
+elementwise arrays validate bindings without execution, whereas scalar reductions
+obey their specified empty identities/errors. `PortableArtifactError` provides `status`, `native_status`, `line`, and
+`column` for malformed artifacts, unsupported requirements, source, binding, and
+evaluation failures. The artifact format is not a sandbox or an arbitrary-input
+numerical-correctness guarantee.
+
+Native dependency packaging is still pending. For development builds, use both
+`FETCHCONTENT_SOURCE_DIR_MINIEXPR` and `MINIEXPR_BUILD_ARTIFACT=ON`, for example:
+
+```sh
+conda run -n blosc2 python -m pip install -e . --no-build-isolation --no-deps \
+  --config-settings=cmake.define.FETCHCONTENT_SOURCE_DIR_MINIEXPR=/path/to/miniexpr \
+  --config-settings=cmake.define.MINIEXPR_BUILD_ARTIFACT=ON
+```
+
+The optional adapter links pinned yyjson only when enabled. Builds without the
+adapter remain usable and report `NotImplementedError` for artifact operations,
+without attempting Python validation/execution as a fallback. The native candidate
+format and shared fixtures live in miniexpr's `doc/dsl-spec/artifact-1.0.md` and
+`tests/portable-artifacts/`.
+
 `@blosc2.jit` auto-detects this DSL: a decorated function whose body contains an
 `if`/`for`/`while` and that compiles under this grammar is dispatched here
 automatically, so its branches and loops actually run, once per chunk, instead

doc/reference/index.rst

@@ -12,4 +12,6 @@ API Reference
     save_load
     utilities
     dsl_syntax
+    portable_dsl
+    menudet_accuracy
     jit

doc/reference/jit.rst

@@ -145,6 +145,12 @@ accept them through ``**kwargs``. Eligible lazy-expression reductions, such as
 settings tune execution rather than array storage; they are not options to
 :func:`blosc2.empty`.
 
+Scalar miniexpr reductions currently use the native interpreter, even when JIT
+is requested. Elementwise DSL returns (including automatically lifted expressions)
+write a block of values and cannot safely target a single reduction accumulator.
+Other execution options still apply, and non-scalar reduction paths retain their
+usual evaluation policies.
+
 ``jit=None`` (default)
     Use the entry point's default policy. DSL kernels, including the real-valued
     ramps used by ``arange`` and ``linspace``, try JIT. Plain expressions do not
@@ -268,9 +274,9 @@ controls have different scopes; neither is an indication that JIT actually ran.
       - Disable applicable process-local positive-cache reuse. This does not
         disable persistent CC disk-cache reuse.
     * - ``ME_DSL_JIT_COMPILER``
-      - Advanced miniexpr-wide compiler override, ``tcc`` or ``cc``. It overrides
-        native compiler selection in DSL pragmas, including the selection emitted
-        by Python's ``jit_backend`` option. Prefer per-call options ordinarily.
+      - Advanced miniexpr-wide compiler default, ``tcc`` or ``cc``. It overrides
+        Python's ``jit_backend`` selection, but an explicit author-written
+        ``# me:compiler=tcc|cc`` pragma remains authoritative.
 
 Tracing and restricted environments
 -----------------------------------