diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 868b01763..43c7ea405 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,6 +108,17 @@ jobs: PYTEST_DEADMAN_SECONDS: "300" PYTEST_DEADMAN_SESSION_SECONDS: "1500" + - name: Certify bounded Menudet math against independent references + run: python scripts/certify_menudet_math.py --report menudet-math.json + + - name: Upload Menudet math certification + if: always() + uses: actions/upload-artifact@v4 + with: + name: menudet-math-${{ matrix.os }}-py${{ matrix.python-version }}-np${{ matrix.numpy-version || 'default' }} + path: menudet-math.json + if-no-files-found: ignore + # TUIs are timing-sensitive and only boot a headless terminal: one OS is # enough, and a flake here is feedback, not a release blocker. - name: Test TUI (non-blocking) diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml index ef3c26092..14be8aa4b 100644 --- a/.github/workflows/cibuildwheels.yml +++ b/.github/workflows/cibuildwheels.yml @@ -1,6 +1,8 @@ name: Python wheels on: + # Certify development revisions without creating a release tag or publishing. + workflow_dispatch: # Trigger the workflow only for tags and PRs to the main branch push: tags: @@ -11,6 +13,20 @@ on: env: CIBW_BUILD_VERBOSITY: 1 + # Every tested wheel checks portable descriptors/persistence and independent + # numeric references, not only whether the extension can be imported. + # Free-threaded CPython safely enables the GIL for this extension. Allow only + # that expected import warning; never force PYTHON_GIL=0 or claim no-GIL safety. + CIBW_TEST_COMMAND: >- + python -m pytest + {project}/tests/test_portable_artifact.py + {project}/tests/test_portable_descriptor.py + {project}/tests/test_portable_descriptor_host.py + {project}/tests/test_portable_lazy.py + {project}/tests/test_portable_policy.py + {project}/tests/test_portable_table_export.py + {project}/tests/test_dsl_portable.py -q -n 0 && + python {project}/scripts/certify_menudet_math.py # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long # Now that github provides native arm64 runners, we can enable tests again # CIBW_TEST_SKIP: "*linux*aarch64*" @@ -156,7 +172,9 @@ jobs: CIBW_BUILD: ${{ matrix.cibw_build }} # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF" - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: >- + pytest {project}/tests && + python {project}/scripts/certify_menudet_math.py # Pin the Pyodide version explicitly per target for reproducible builds. # Note: in-memory SChunk get_slice is broken on the Pyodide 0.29.x # Emscripten toolchain (works on 314); see gh-664. The affected test is @@ -282,6 +300,9 @@ jobs: python -m pytest tests/test_open.py tests/test_vlmeta.py \ tests/ndarray/test_evaluate.py -q -p no:cacheprovider + - name: Certify Menudet on the installed abi3 wheel + run: python scripts/certify_menudet_math.py + upload_pypi: needs: [ build_wheels, build_wheels_wasm, test_abi3_matrix ] diff --git a/.github/workflows/wasm-tests.yml b/.github/workflows/wasm-tests.yml index c9ef42595..211506b06 100644 --- a/.github/workflows/wasm-tests.yml +++ b/.github/workflows/wasm-tests.yml @@ -19,7 +19,9 @@ jobs: # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF" # cibuildwheel runs the suite automatically in the node/pyodide runtime - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: >- + pytest {project}/tests && + python {project}/scripts/certify_menudet_math.py steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 diff --git a/CMakeLists.txt b/CMakeLists.txt index e9c5a078b..e31cd81f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,7 @@ add_custom_command( COMMAND Python::Interpreter -m cython "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx" --output-file blosc2_ext.c DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx" + "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/dsl_artifact_bridge.h" VERBATIM) add_custom_command( @@ -117,6 +118,7 @@ set(MINIEXPR_BUILD_SHARED OFF CACHE BOOL "Build miniexpr shared library" FORCE) set(MINIEXPR_BUILD_TESTS OFF CACHE BOOL "Build miniexpr tests" FORCE) set(MINIEXPR_BUILD_EXAMPLES OFF CACHE BOOL "Build miniexpr examples" FORCE) set(MINIEXPR_BUILD_BENCH OFF CACHE BOOL "Build miniexpr benchmarks" FORCE) +option(MINIEXPR_BUILD_ARTIFACT "Build portable DSL artifact support" ON) # Keep miniexpr's and C-Blosc2's bundled files inside the blosc2 package. # Without this, their install rules use the default CMAKE_INSTALL_LIBDIR # ("lib"), which scikit-build places at site-packages/lib. @@ -140,13 +142,20 @@ endif() FetchContent_Declare(miniexpr GIT_REPOSITORY https://github.com/Blosc/miniexpr.git - GIT_TAG 3f4db93a628aedc662b7d27a5f35c97ffaeb0424 + GIT_TAG 3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4 # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr ) FetchContent_MakeAvailable(miniexpr) # Link against miniexpr static library target_link_libraries(blosc2_ext PRIVATE miniexpr_static) +if(TARGET miniexpr_artifact) + target_link_libraries(blosc2_ext PRIVATE miniexpr_artifact) + target_compile_definitions(blosc2_ext PRIVATE B2_HAVE_PORTABLE_ARTIFACT=1) + FetchContent_GetProperties(yyjson SOURCE_DIR B2_YYJSON_SOURCE_DIR) + install(FILES "${B2_YYJSON_SOURCE_DIR}/LICENSE" + DESTINATION ${SKBUILD_METADATA_DIR}/licenses/miniexpr RENAME LICENSE-YYJSON) +endif() if(APPLE) target_link_libraries(blosc2_ext PRIVATE "-framework Accelerate") endif() diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1d7bd308c..7bcdb6756 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,49 @@ # Release notes +## Changes from 4.14.2 to 4.15.0 (in development) + +### Introducing the Menudet 1.0 draft + +**Menudet — a little language for portable computation on arrays and tables.** + +- Export normalized `DSLKernel` authoring as validated, standalone JSON artifacts + with typed signatures and exact captured constants. Native miniexpr executes + imports without reconstructing Python functions or falling back to Python. +- Support checked integer arithmetic, operand-typed floating computation, control + flow, fixed-width strings, logical array coordinates and ordered block reductions. +- Keep logical reduction groups stable through persistence, partial reads and + output rechunking. True block-scalar results expose the logical block-grid shape. +- Validate kernel inputs to `CTable.add_computed_column()` and + `add_generated_column()` as Menudet artifacts with implicit independent-row + semantics, including fixed-shape row reductions. Named bindings are supported; + positional bindings and column-bound DSL LazyUDFs adopt the same semantics. + New cross-row computations must use array partitions or table aggregation, + rather than implicit execution blocks or append batches. +- Remove the draft `add_portable_*` column methods and `row_domain` argument. + Fixed-shape row reductions use `cardinality="block_scalar"`. +- New compliant DSLKernel-backed LazyUDF saves export the draft automatically; + unsupported saves fail validation before writing the destination. +- Historical Python-specific recipes require explicit `deserialize="full"`. + MessagePack decoding is safe by default, including nested containers. +- Saved legacy table-kernel recipes retain their old block/batch-dependent + behavior only with trusted, explicit `deserialize="full"`; safe-mode errors + explain the opt-in. Recipes are never automatically reinterpreted. +- Ordinary LazyExpr persistence retains supported numerical calls, reductions, + methods and indexing under default loading, with existing expression validation. + Full-only nested payloads remain gated. String/lazy expressions and non-kernel + row transformers retain their existing table registration behavior. + +Menudet 1.0 is a **draft**, not a frozen compatibility promise. The existing +`DSLKernel` and portable API names remain unchanged. This draft uses the typed +interpreter; optional JIT requests fall back before execution, while required JIT +rejects unsupported acceleration. Complex computation, callbacks, arbitrary +memory access and non-strict floating-point modes are outside the portable contract. + +See the [Menudet reference](https://www.blosc.org/python-blosc2/reference/portable_dsl.html) +for array/table examples, persistence policy and current limitations. Release +certification is tracked separately; this development entry is not an announcement +that 4.15.0 has been published. + ## Changes from 4.14.1 to 4.14.2 XXX version-specific blurb XXX diff --git a/doc/conf.py b/doc/conf.py index 915bb0c20..b709da445 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -111,7 +111,14 @@ def genbody(f, func_list, lib="blosc2"): ], } -exclude_patterns = ["_build", ".DS_Store", "**.ipynb_checkpoints", "tutorials/images/**"] +exclude_patterns = [ + "_build", + ".DS_Store", + "**.ipynb_checkpoints", + "tutorials/images/**", + "html/**", + "**/autofiles-stash/**", +] html_show_sourcelink = False @@ -306,7 +313,7 @@ def process_sig(app, what, name, obj, options, signature, return_annotation): # ``-W`` (as CI does) to turn that warning into a failure. _AUTODOC_DIRECTIVE = re.compile( - r"^\s*\.\.\s+(?:autoclass|autofunction|autodata|autoexception|autodecorator)::" + r"^\s*\.\.\s+(?:autoclass|autofunction|autodata|autoexception|autodecorator|py:class)::" r"\s*([\w.]+)" ) _CURRENTMODULE = re.compile(r"^\s*\.\.\s+(?:currentmodule|module)::\s*([\w.]+)") diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index fd0e689de..7ba918308 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -33,13 +33,13 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax: - The :doc:`b2view <../guides/b2view>` terminal browser (``textual``, ``textual-plotext``), including its in-terminal braille plot (the ``p`` key). Required by the ``b2view`` command. - * - ``hires`` + * - ``hires`` - The high-resolution image view in b2view (the ``h`` key), which renders a real ``matplotlib`` image in the terminal - (``textual-image``, ``matplotlib``). Includes ``tui``. - * - ``images`` - - JPEG/PNG ordinary-file previews in b2view (``Pillow``, ``textual-image``). - Includes ``tui`` without requiring matplotlib. ``hires`` includes this extra. + (``textual-image``, ``matplotlib``). Includes ``tui``. + * - ``images`` + - JPEG/PNG ordinary-file previews in b2view (``Pillow``, ``textual-image``). + Includes ``tui`` without requiring matplotlib. ``hires`` includes this extra. * - ``parquet`` - The ``parquet-to-blosc2`` converter (``pyarrow``); see :doc:`../guides/parquet_to_blosc2`. diff --git a/doc/reference/dsl_syntax.md b/doc/reference/dsl_syntax.md index 2568d3171..71e22deb5 100644 --- a/doc/reference/dsl_syntax.md +++ b/doc/reference/dsl_syntax.md @@ -3,6 +3,10 @@ This is the practical reference for the DSL used by `@blosc2.dsl_kernel` functions (and checked by `blosc2.validate_dsl()`). It focuses on what works today and the most common gotchas. +The portable subset is named **Menudet**, a little language for portable +computation on arrays and tables. See the [Menudet reference](portable_dsl.rst) +for the 1.0 draft contract. This page also documents ordinary full-DSL features; +acceptance here does not imply portability. For usage walkthroughs and end-to-end examples, see the [LazyArray UDF DSL kernels tutorial](../tutorials/03.lazyarray-udf-kernels.ipynb). @@ -18,6 +22,139 @@ def kernel(x, y): Use Python-style indentation and always return a value on the paths you execute. +### Native source without a Python function + +`blosc2.DSLKernel.from_source(source)` constructs a native-only kernel without +executing source as Python or inspecting Python globals: + +```python +source = """# me:compiler=tcc +def affine(x): + return x * 2.0 + 1.0 +""" +kernel = blosc2.DSLKernel.from_source(source) +result = blosc2.lazyudf(kernel, (x,), dtype="float64")[:] +``` + +The source and header pragmas are preserved verbatim. Input dtypes come from the +supplied arrays; pass an explicit output dtype. The native compiler validates the +body with these types during computation. The constructor inspects only the +plain positional function header and does not certify portable-profile support. +Frontend conveniences, captured globals, and Python-function fallback are not +available on this path; use native function spellings such as `sin(x)`. + +This is separate from the existing Python-based persistence reconstruction path. +Portable artifact export/import is still under development. The full native +language reference and experimental portable profile live in miniexpr's +`doc/dsl-syntax.md` and `doc/dsl-spec/1.0.md`, respectively. The portable profile +and artifact contract are experimental drafts; see [portable artifacts](portable_dsl.rst) +for the release boundary and legacy Python persistence policy. + +### Experimental portable-profile validation + +```python +info = blosc2.validate_portable_dsl(source, {"x": "float64"}, "float64") +if not info["valid"]: + print(info["status"], info["line"], info["column"], info["error"]) +``` + +This native-only check uses raw source and explicit logical dtypes, with no +frontend rewriting or captured globals. It neither executes the kernel nor +invokes a JIT compiler. It reports version, signature, source, and unsupported +feature errors separately. Potential runtime errors, including missing returns +on some paths, do not automatically invalidate a program. + +Draft 1.0 admits Boolean, standard integer widths, float32/float64 and fixed strings, +mixed operand-driven numeric typing, checked arithmetic, reductions and explicit ND context. +It is separate from the ordinary full language described below. +Validation is not a sandbox, backend-support probe, or a guarantee about +arbitrary runtime data. Older native builds report `runtime_unsupported` rather +than attempting Python validation. Native dependency integration for this +experimental API is pending; development builds can use the local miniexpr source +via CMake's `FETCHCONTENT_SOURCE_DIR_MINIEXPR` override. + +### Portable artifact export and import (experimental) + +Export with explicit input/output dtypes. Scalar globals and closure values are +snapshotted into typed manifest constants, not inserted as Python literals: + +```python +SCALE = 2.0 + + +@blosc2.dsl_kernel +def scaled(x): + return x * SCALE + + +artifact = scaled.export({"x": "float64"}, "float64") +SCALE = 99.0 # Does not change the saved artifact. +kernel = blosc2.PortableKernel.from_json(artifact, jit=False) +values = kernel.evaluate({"x": np.arange(4, dtype=np.float64)}) +# values: array([0., 2., 4., 6.]) +``` + +`export()` returns deterministic UTF-8 JSON text; save it with an ordinary file +write. `from_json()` accepts text or bytes, not a pathname or decoded mapping. +Import uses the same native JSON loader as C and does not need the author's module, +function, decorators, globals, Python source reconstruction, or frontend rewrites. +`evaluate()` returns a new NumPy array; `blosc2.asarray(values)` can materialize +compressed storage. `kernel.lazy()` binds explicit logical partitions for lazy +evaluation and portable persistence. Newly authored compliant DSLKernel-backed +LazyUDF saves also normalize and validate portable native recipes automatically. + +Python `bool`, `int`, and `float` captures infer `bool`, `int64`, and `float64`. +Supported NumPy scalar types retain their dtype. Use +`capture_dtypes={"SCALE": "float32"}` to explicitly specialize a capture; finite +float narrowing and explicit integer-to-float conversion round to the chosen dtype; +out-of-range values reject. Arrays (including zero-dimensional arrays), objects, unsupported +scalar types, unresolved names, and external callbacks cannot be captures. +Export does not call the kernel or capture conversion hooks. It reuses supported +authoring normalization, such as `np.sin(x)`, before native profile validation. +Fixed byte/Unicode scalar snapshots are supported; arbitrary objects and callbacks are not. + +Native-only kernels can bind existing parameters as constants explicitly: + +```python +author = blosc2.DSLKernel.from_source( + "# me:compiler=cc\ndef scaled(x, scale):\n return x * scale\n" +) +artifact = author.export({"x": "float64"}, "float64", constants={"scale": 2.0}) +``` + +Every source parameter requires exactly one runtime input or constant binding. +Parameters and captures may have different supported dtypes; computation is +operand-driven before output conversion. Manifest float constants preserve +IEEE bit patterns, including signed zero and non-finite values; integer constants +are range-checked decimal strings. Comments and retained native pragmas survive +parameterization. Required strict FP semantics are independent of host defaults; +compiler pragmas remain preferences. Draft 1.0 currently always uses the typed interpreter. + +Runtime arrays are bound by name in any mapping order and must have the exact +declared logical dtype and common shape. The adapter converts endian order, +alignment, and strides, but does not broadcast arrays or silently change their +dtype. Constant-only kernels require `evaluate({}, shape=(...))`; empty +elementwise arrays validate bindings without execution, whereas scalar reductions +obey their specified empty identities/errors. `PortableArtifactError` provides `status`, `native_status`, `line`, and +`column` for malformed artifacts, unsupported requirements, source, binding, and +evaluation failures. The artifact format is not a sandbox or an arbitrary-input +numerical-correctness guarantee. + +Native dependency packaging is still pending. For development builds, use both +`FETCHCONTENT_SOURCE_DIR_MINIEXPR` and `MINIEXPR_BUILD_ARTIFACT=ON`, for example: + +```sh +conda run -n blosc2 python -m pip install -e . --no-build-isolation --no-deps \ + --config-settings=cmake.define.FETCHCONTENT_SOURCE_DIR_MINIEXPR=/path/to/miniexpr \ + --config-settings=cmake.define.MINIEXPR_BUILD_ARTIFACT=ON +``` + +The optional adapter links pinned yyjson only when enabled. Builds without the +adapter remain usable and report `NotImplementedError` for artifact operations, +without attempting Python validation/execution as a fallback. The native candidate +format and shared fixtures live in miniexpr's `doc/dsl-spec/artifact-1.0.md` and +`tests/portable-artifacts/`. + `@blosc2.jit` auto-detects this DSL: a decorated function whose body contains an `if`/`for`/`while` and that compiles under this grammar is dispatched here automatically, so its branches and loops actually run, once per chunk, instead diff --git a/doc/reference/index.rst b/doc/reference/index.rst index a0ba44c75..6fc2636c6 100644 --- a/doc/reference/index.rst +++ b/doc/reference/index.rst @@ -12,4 +12,6 @@ API Reference save_load utilities dsl_syntax + portable_dsl + menudet_accuracy jit diff --git a/doc/reference/jit.rst b/doc/reference/jit.rst index ecae2de28..49eeb939f 100644 --- a/doc/reference/jit.rst +++ b/doc/reference/jit.rst @@ -145,6 +145,12 @@ accept them through ``**kwargs``. Eligible lazy-expression reductions, such as settings tune execution rather than array storage; they are not options to :func:`blosc2.empty`. +Scalar miniexpr reductions currently use the native interpreter, even when JIT +is requested. Elementwise DSL returns (including automatically lifted expressions) +write a block of values and cannot safely target a single reduction accumulator. +Other execution options still apply, and non-scalar reduction paths retain their +usual evaluation policies. + ``jit=None`` (default) Use the entry point's default policy. DSL kernels, including the real-valued ramps used by ``arange`` and ``linspace``, try JIT. Plain expressions do not @@ -268,9 +274,9 @@ controls have different scopes; neither is an indication that JIT actually ran. - Disable applicable process-local positive-cache reuse. This does not disable persistent CC disk-cache reuse. * - ``ME_DSL_JIT_COMPILER`` - - Advanced miniexpr-wide compiler override, ``tcc`` or ``cc``. It overrides - native compiler selection in DSL pragmas, including the selection emitted - by Python's ``jit_backend`` option. Prefer per-call options ordinarily. + - Advanced miniexpr-wide compiler default, ``tcc`` or ``cc``. It overrides + Python's ``jit_backend`` selection, but an explicit author-written + ``# me:compiler=tcc|cc`` pragma remains authoritative. Tracing and restricted environments ----------------------------------- diff --git a/doc/reference/menudet_accuracy.md b/doc/reference/menudet_accuracy.md new file mode 100644 index 000000000..70bb2f189 --- /dev/null +++ b/doc/reference/menudet_accuracy.md @@ -0,0 +1,52 @@ +# Menudet draft: numeric certification + +Menudet requires checked integer results and the stated floating-point type, +domain, NaN, infinity, signed-zero and rounding rules. It does **not** promise +bit-identical transcendental results across host math libraries or a universal +ULP error bound. Native exact-anchor and exceptional-value tests remain separate +from the bounded independent math check described here. + +## Reproducible independent references + +`scripts/menudet_math_vectors.json` contains 284 samples for 40 canonical +operations, covering both float32 and float64. Inputs and expected results are +stored as exact hexadecimal encodings. References were generated with mpmath +1.3.0 at both 100 and 200 decimal digits; regeneration rejects disagreement +between the two rounded results. The checker does not need mpmath and does not +use NumPy or the executing host's libm as its mathematical reference. + +The finite domains include inverse-function endpoints, small arguments that +exercise cancellation, mixed quadrants, gamma functions, and explicit fused +multiply-add cancellation. Domain errors, NaNs/infinities, subnormal behavior, +signed zeros, rounding functions, nextafter and aliases additionally have native +conformance fixtures; they are not certified merely by these finite samples. + +| Operations | Maximum error for the checked samples | +| --- | --- | +| cospi, sinpi, exp10, lgamma, tgamma, logaddexp | 8 ULP | +| Other sampled transcendental functions, sqrt, hypot, floating pow | 4 ULP | +| fdim, fmin, fmax, copysign, fmod, remainder, ldexp, fma | Exact rounded value | + +These are **function-specific acceptance budgets for this finite corpus**, not +new normative bounds over all possible inputs. ULP is measured using the smaller +adjacent gap at the expected value, including at powers of two. Adding a new +sample or changing a budget requires review; a failed platform must not silently +skip a function or inflate its tolerance. + +## Running certification + +Against an installed package: + +```console +python scripts/certify_menudet_math.py --report menudet-math.json +``` + +Reports identify the package path/version, platform, reference generator, sample +count, per-function/per-precision maxima and individual failures. The process +exits nonzero on failure. Use `--generate` only to regenerate the checked-in +reference corpus in a development environment with mpmath installed. + +Native Python CI uploads the reports. The same corpus runs in Pyodide and in the +installed-wheel test command, without adding a runtime reference-library +dependency. Passing a local wheel certifies only that tested wheel and host; +release readiness additionally requires the supported platform matrix to pass. diff --git a/doc/reference/misc.rst b/doc/reference/misc.rst index e9ce82962..e91d413bf 100644 --- a/doc/reference/misc.rst +++ b/doc/reference/misc.rst @@ -7,7 +7,10 @@ This page documents the miscellaneous members of the ``blosc2`` module that do n .. autodata:: cpu_info -.. autoclass:: finfo +.. py:class:: finfo(dtype) + + Return floating-point limits for ``dtype``. This is an alias for + :class:`numpy.finfo`; see NumPy's reference for its attributes and examples. .. autoclass:: iinfo diff --git a/doc/reference/portable_dsl.rst b/doc/reference/portable_dsl.rst new file mode 100644 index 000000000..bd9d1035d --- /dev/null +++ b/doc/reference/portable_dsl.rst @@ -0,0 +1,222 @@ +Menudet: portable computation +============================= + +**Menudet — a little language for portable computation on arrays and tables.** + +Menudet is the language; miniexpr is its native implementation, and Python-Blosc2 +provides authoring, array scheduling and table integration. The existing +``DSLKernel``, ``PortableKernel`` and ``validate_portable_dsl`` API names remain +unchanged. Python-like authoring syntax normalizes to a language-independent +artifact; a Menudet artifact is not an executable Python recipe. + +.. warning:: + + Menudet 1.0 is experimental. Its language and artifact specifications are + drafts, not a frozen compatibility promise. Native cross-platform CI passes; + independent numeric accuracy and Python release-wheel certification are + tracked separately. Beta feedback will inform the final compatibility contract. + +Menudet artifacts use the versioned native 1.0 profile. +Artifacts are standalone UTF-8 JSON, executable through native miniexpr in C or +Python, without reconstructing or executing Python functions on import. + +.. code-block:: python + + import blosc2 + import numpy as np + + author = blosc2.DSLKernel.from_source("def affine(x):\n return x * 2.0 - 1.0\n") + artifact = author.export({"x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + values = kernel.evaluate({"x": np.arange(4, dtype="float64")}) + # array([-1., 1., 3., 5.]) + +The result is an eager NumPy array. Bindings are named and require exact dtypes +and matching shapes. Supported dtypes are Boolean, standard signed/unsigned +integer widths, ``float32``, ``float64`` and fixed-width bytes/Unicode strings; +parameters may have different dtypes. Captured Python/NumPy scalars are snapshotted into +typed constants, never live callbacks or arrays. Integers use range-checked +decimal strings; floats use exact IEEE-754 hexadecimal bits. + +For a runnable array/table example using the same kernel, see +``examples/menudet.py`` in the source distribution. Numeric certification and its +finite-domain limits are described in :doc:`menudet_accuracy`. + +Release boundary +---------------- + +``DSLKernel.export`` and ``validate_portable_dsl`` default to draft 1.0. +Native validation of normalized source, typed captures, +fixed widths and cardinality completes before export returns JSON:: + + author = blosc2.DSLKernel.from_source("def total(x):\n return sum(x)\n") + record = author.export({"x": "int64"}, "int64", + cardinality="block_scalar") + kernel = blosc2.PortableKernel.from_json(record) + values = kernel.lazy({"x": np.arange(6, dtype="int64")}, partitions=(3,))[:] + # array([3, 12]) + +The draft admits all standard Boolean/integer widths, float32/float64, fixed byte +and Unicode strings, mixed operand-driven numeric promotion, checked arithmetic, +native math/string operations, masked ordered block reductions and logical ND +coordinates. Specify ``ndim`` for coordinate-bearing sources. Existing static +``row["column"]``, NumPy calls and string syntax reuse the full-DSL authoring +normalizers; dynamic indexing, objects and callbacks are not reconstructed. + +Raw draft source can also be checked without artifacts, arrays, execution or JIT:: + + info = blosc2.validate_portable_dsl( + "def coordinate(x):\n return x + _i0\n", + {"x": "int16"}, "int64", ndim=1, + cardinality="elementwise") + assert info["valid"] + +The core native validator checks exact numeric/string signatures and reports +native status and source line/column. String dtypes specify exact slot widths, +for example ``{"x": "U4"}``; output widths must match native inference. +``cardinality=None`` checks inferred cardinality; an explicit declaration must +match. Reserved ND symbols require an explicit covering ``ndim``. This source +validator remains available when the optional artifact JSON loader is disabled. +It checks raw native syntax, not Python normalization, globals or runtime domains. + +Supported features include elementwise arithmetic, comparisons and chains, +Boolean logic, local assignments, branches, bounded loops, and explicit output +conversions. Boolean operands participate in numeric arithmetic as zero/one; +truth conversion occurs at a Boolean output or explicit ``bool()`` call. + +External functions, callbacks, object operations and unsupported dynamic indexing +are rejected during export and independently by native import; there is no Python fallback. + +Integer overflow, out-of-range narrowing, non-finite/out-of-range float-to-integer +casts and integral zero divisors raise checked evaluation errors. The interpreter +establishes its floating-point environment and restores the caller's environment. +Callers must keep data and intermediates in the defined domains; +static validation cannot prove those conditions. This is not a sandbox for +untrusted source or arbitrary data. NaN payloads are not guaranteed. + +Use ``blosc2.validate_portable_dsl(source, input_dtypes, output_dtype)`` to check +membership without execution or JIT. It returns validity, a status category, +and optional source diagnostics. Import/export errors raise +``blosc2.PortableArtifactError`` with a stable ``status`` attribute. Missing +returns, zero range steps, and loop-cap failures are runtime evaluation errors; +failed output contents are unspecified. Empty elementwise evaluations validate +bindings without execution; block-scalar reductions use their specified empty identities/errors. + +Builds and execution +-------------------- + +Draft logical partitions and persistence +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a validated 1.0 kernel, ``kernel.lazy(inputs, partitions=(...))`` binds an +immutable logical C-order block grid. This grid is independent of compressed +operand chunks/blocks. Elementwise output retains the original domain; +``block_scalar`` output has one item per logical partition (the grid shape). +Scalar returns under lane-varying branches/loops, or inside loops with +lane-varying break/continue, reject as ambiguous output—even if their scalar +expressions match. Uniform reduction conditions and coherent scalar control +flow remain supported; varying loops may rejoin before a top-level scalar return. +Slices evaluate intersecting **original complete groups** before selecting result +lanes. ND coordinates retain the original domain and group origin. Rechunking the +materialized result changes storage, never reduction membership:: + + lazy = kernel.lazy({"x": stored_array}, partitions=(32, 64)) + values = lazy[1:3, ::2] + result = lazy.compute(chunks=(8, 16), blocks=(4, 8)) + lazy.save("recipe.b2nd") + reopened = blosc2.open("recipe.b2nd") + +Native JSON records, signatures and logical partitions persist through disk +carriers, frames, structured MessagePack and EmbedStore/DictStore/TreeStore, +including arithmetic expressions containing portable carriers. Operands must +have persistent references; failures preflight before overwriting destinations. +Import never reconstructs a Python function. Safe arithmetic envelopes permit +only bound-name primitive arithmetic/Boolean expressions, not calls or attributes. + +New DSLKernel-backed LazyUDF saves automatically normalize authoring and typed +captures, validate draft 1.0 natively and persist artifact/bindings/context. The +same preflight applies to frames, MessagePack, stores and nested arithmetic +references. Python UDFs and noncompliant normalized kernels reject before +destination writes; there is no source fallback. Positional scalar arguments +become typed constants, array arguments retain their parameter names, and the +current output shape and logical block partitions are preserved. A block-scalar +return cannot silently replace an elementwise LazyUDF shape: author it through +``export(cardinality="block_scalar")`` and ``PortableKernel.lazy`` +with explicit partitions instead. Storage chunk boundaries must not split the +declared uniform logical block grid. + +Historical source-backed files require explicit ``deserialize='full'``; +that policy propagates through nested references. Loaded legacy recipes are not +automatically migrated by saving. CTable legacy computed/generated DSL metadata +has the same explicit policy gate. The older table vector-column DSL API lacks +an explicit row-domain contract and cannot be silently converted to independent +rows when loading. New registrations use the independent-row contract below. +``msgpack_unpackb()`` also defaults to safe decoding. Active reconstruction needs +explicit ``deserialize='full'``; container readers propagate their effective +policy through nested values instead of relying on the helper's default. + +Draft native table transformers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Scalar stored columns bind to native transformers by name:: + + table.add_computed_column("total", kernel, inputs={"x": "amount"}) + table.add_generated_column("stored_total", values=kernel, + inputs={"x": "amount"}) + +These APIs also accept a newly authored ``DSLKernel`` directly. Specify the +output dtype when it differs from input type promotion. Normalization, capture snapshots and native validation finish +before column registration or destination mutation:: + + table.add_computed_column( + "twice", blosc2.DSLKernel.from_source("def twice(x):\n return x * 2\n"), + inputs={"x": "amount"}, dtype="int64") + +For a fixed-shape row reduction, additionally specify +``cardinality="block_scalar"``. Independent-row grouping is the contract of +kernel registration; there is no ``row_domain`` option or separate +``add_portable_*`` API. Positional input lists and column-bound DSL LazyUDFs +also adopt this contract on new registration. String/lazy expressions and +non-kernel row transformers retain their existing behavior. + +The independent domain means **each scalar row is one original +one-lane group**, not a storage block or a changing table-wide reduction. Rank-one +coordinates are row-local: shape ``(1,)``, origin ``(0,)``, ``_i0 == 0`` and +``_n0 == 1``. Block-scalar and elementwise kernels both yield one scalar per row. +Named bindings may be empty for constant/ND constructors. Appending, refreshing, +deleting rows, materializing and compact-copying retain this mapping. Artifacts, +bindings and fixed row shapes survive table disk/frame round trips, with native +validation before binding and again before saving. Fixed-width numeric/string +signatures are immutable. Fixed-shape ndarray columns also support scalar native +row reductions: each complete row is one group, its broadcast row shape is the +logical ND domain, and its origin is zero. Scalar columns can broadcast within +that row without changing coordinates. Shape/cardinality incompatibilities reject +before registration. Multi-row table partitions and vector-valued row outputs +are not implicitly admitted. The draft remains subject to conformance and +external platform release gates. + +The native artifact loader is enabled by default in Python package builds. +Custom builds may disable it with ``MINIEXPR_BUILD_ARTIFACT=OFF``; import then +raises ``NotImplementedError``. The optional adapter uses pinned, MIT-licensed +yyjson; the raw native compiler does not depend on it. + +Draft 1.0 uses the typed interpreter. ``jit`` and compiler pragmas are preferences +only and currently fall back to interpretation; a required-JIT artifact rejects. +Semantic evaluation errors never retry the kernel. Artifacts +enforce strict FP independently of host defaults. Host while-loop limits remain +execution policy rather than artifact constants. + +The authoritative language and artifact specifications are maintained in +miniexpr under ``doc/dsl-spec/1.0.md`` and ``doc/dsl-spec/artifact-1.0.md``. +Python-Blosc2 pins the published native revision that passed the native platform +matrix. Native CI is distinct from Python stock-wheel certification. + +API +--- + +.. autoclass:: blosc2.PortableKernel + :members: + +.. autoclass:: blosc2.PortableArtifactError + +.. autofunction:: blosc2.validate_portable_dsl diff --git a/doc/reference/save_load.rst b/doc/reference/save_load.rst index 493447210..ba4e316d4 100644 --- a/doc/reference/save_load.rst +++ b/doc/reference/save_load.rst @@ -21,3 +21,20 @@ Save and load .. autofunction:: save_tensor .. autofunction:: load_tensor .. autofunction:: from_cframe + +Deserialization policy +~~~~~~~~~~~~~~~~~~~~~~ + +Validated Menudet artifacts load natively without reconstructing Python code. +Historical Python-specific recipes require explicit ``deserialize="full"``; +the selected policy propagates through nested containers and references. +Ordinary LazyExpr expressions, including supported numerical calls, reductions, +methods and indexing, load under the default ``deserialize="safe"`` policy +with the existing expression validation. Full-only nested operands (such as +legacy kernels or proxies) still require explicit opt-in. Legacy table-kernel +recipes loaded with ``"full"`` retain their old block/batch-dependent grouping; +new registrations use independent-row Menudet semantics. + +.. autodata:: DeserializeMode + +.. autoexception:: UnsafeDeserializationError diff --git a/examples/menudet.py b/examples/menudet.py new file mode 100644 index 000000000..a44debacd --- /dev/null +++ b/examples/menudet.py @@ -0,0 +1,32 @@ +"""Menudet 1.0 draft: one kernel, array and independent table-row execution.""" + +from dataclasses import dataclass + +import numpy as np + +import blosc2 + +author = blosc2.DSLKernel.from_source("def affine(x):\n return x * 2.0 - 1.0\n") +kernel = blosc2.PortableKernel.from_json(author.export({"x": "float64"}, "float64")) +x = np.arange(6, dtype="float64") +values = kernel.lazy({"x": x}, partitions=(3,))[:] +np.testing.assert_array_equal(values, [-1, 1, 3, 5, 7, 9]) +print("Array:", values) + + +@dataclass +class Row: + amount: float = 0.0 + + +table = blosc2.CTable(Row, new_data={"amount": x.tolist()}, create_summary_index=False) +table.add_computed_column("adjusted", kernel, inputs={"x": "amount"}) +np.testing.assert_array_equal(table["adjusted"][:], values) +print("Table:", table["adjusted"][:]) + +# A reduction is a different result contract: one scalar per ORIGINAL group. +total = blosc2.DSLKernel.from_source("def total(x):\n return sum(x)\n") +record = total.export({"x": "float64"}, "float64", cardinality="block_scalar") +totals = blosc2.PortableKernel.from_json(record).lazy({"x": x}, partitions=(3,))[:] +np.testing.assert_array_equal(totals, [3, 12]) +print("Block totals:", totals) diff --git a/examples/ndarray/portable_artifact.py b/examples/ndarray/portable_artifact.py new file mode 100644 index 000000000..363bcdd9f --- /dev/null +++ b/examples/ndarray/portable_artifact.py @@ -0,0 +1,25 @@ +####################################################################### +# Copyright (c) 2026, Blosc Development Team +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### +"""Snapshot a scalar capture and execute the portable artifact without its author.""" + +import numpy as np + +import blosc2 + +SCALE = 2.0 + + +@blosc2.dsl_kernel +def scaled(x): + return x * SCALE + + +if __name__ == "__main__": + artifact = scaled.export({"x": "float64"}, "float64") + SCALE = 99.0 + imported = blosc2.PortableKernel.from_json(artifact, jit=False) + values = imported.evaluate({"x": np.arange(4, dtype=np.float64)}) + np.testing.assert_array_equal(values, [0, 2, 4, 6]) + print(values) diff --git a/plans/portable-DSL-1.0-implemenation.md b/plans/portable-DSL-1.0-implemenation.md new file mode 100644 index 000000000..a01531f31 --- /dev/null +++ b/plans/portable-DSL-1.0-implemenation.md @@ -0,0 +1,503 @@ +# Portable DSL 1.0 implementation plan + +Date: 2026-10-07. Status: implementation plan; all three detail decisions +below are confirmed by the user. Feature scope and earlier semantic decisions are +approved in `portable-DSL-1.0-scope-inventory.md`. This file intentionally uses +the requested filename. + +## Objective and shortest implementation path + +Deliver a public, versioned kernel contract executable from C and Python, with +portable-only new kernel persistence. Build correctness in the native +**interpreter first**. Reuse existing parsing, expression evaluation, strings, +ND support and artifact loading; do not implement a second language engine. + +Initially route every 1.0 kernel through the interpreter. Subsequently enable +only existing JIT paths whose semantics are demonstrated compatible. Missing +JIT support, including checked arithmetic, masked reductions or strings, is +never a reason to delay interpreter delivery or weaken the contract. + +Draft portable 1.0 is the sole public portable format and the authoring/validation +default. The unreleased 0.1 profile is retired, with explicit unsupported-version +rejection rather than migration or reinterpretation. Ordinary full-DSL +execution keeps its existing semantics. Miniexpr owns normative semantics, +validation and conformance; Python-Blosc2 owns normalization and host integration. + +## 0. Confirmed behavioral choices + +The user approved Q1–Q3 below. Their recommended rules are now implementation +requirements; the explanations preserve the rationale for these choices. + +### Q1. Complete the numeric type policy + +Recommend Boolean, signed/unsigned 8/16/32/64-bit integers, and float32/float64. +No float16, complex, object, datetime or structured arithmetic in 1.0. + +Proposed promotion rules: + +- Same typed operands retain their type; Boolean arithmetic uses int64 numeric + zero/one, with `/` producing float64. Boolean logical operations remain Boolean. +- Same-signedness integers widen to the wider operand. Mixed signed/unsigned + integers use the smallest signed type covering both complete operand ranges. + Reject an implicit combination for which no such type exists, notably + int64/uint64; explicit float conversion remains possible with documented loss. +- float32 with float32 stays float32; float64 dominates. Integer/float mixtures + use float32 for integers of at most 16 bits and float64 otherwise. Conversion + to float64 may round 64-bit integers; document this rather than claiming exactness. +- Weak source literals adopt an accompanying typed operand when within range. + Decimal floating literals round once to the selected IEEE type using nearest, + ties-to-even; do not require exact binary representability of `0.1`. + An out-of-range contextual literal is rejected, requiring an explicit cast or + typed constant. Literal-only integers default to int64 and floats to float64. +- Typed captured constants are strong operands, not weak literals. `int()` yields + int64, `float()` yields float64, and `bool()` yields Boolean. Explicit typed + casts, if admitted in the builtin matrix, have precisely named destination types. +- Arithmetic promotion does not define comparisons by accident: compare mixed + signed/unsigned integers exactly without lossy float intermediates. Reject any + unsupported mixed comparison explicitly in the initial matrix. + +Why approval matters: uint64 is useful, but there is no exact fixed-width signed +common type with int64. This policy chooses a diagnostic rather than implicit +float64 rounding for their arithmetic. Narrow integer arithmetic can overflow +before final widening, consistently with operand-driven computation. + +### Q2. Represent block-scalar outputs in the lazy host + +Recommend explicit result cardinality in the artifact: `elementwise` or +`block_scalar`. A standalone C block evaluation returns one value for the latter. +In Python lazy execution, expose one value per logical evaluation block, on the +block-grid shape, with row-major block ordering. Do not broadcast an actual +block-scalar return back to every element implicitly. + +A scalar reduction used inside an elementwise calculation broadcasts to its +participating lanes; that does not itself make the kernel block-scalar. +Reject paths with inconsistent return cardinality. A block-scalar partial read +indexes output blocks; an elementwise partial read evaluates original input +groups before selecting lanes. Empty input arrays have no scheduled blocks; +an explicitly evaluated empty standalone block still obeys reduction identities. + +Why approval matters: today's artifact API assumes `nitems` output values. +Silently keeping that allocation model would confuse scalar reductions with +elementwise results and obscure what a saved array's shape means. + +### Q3. Reuse the existing legacy opt-in + +Recommend user-supplied `deserialize="full"` as the legacy-kernel opt-in, rather +than adding another public switch. Normal/default safe opens must not reconstruct +legacy Python kernels. Propagate the effective policy through nested operands; +internal calls that hard-code `full` must not grant permission on the user's +behalf. If existing API defaults cannot distinguish explicit permission from an +implicit default, adjust the boundary/policy propagation before admitting legacy +execution. Portable artifacts need native validation, not Python reconstruction. + +No migration tool, automatic conversion, or legacy-save fallback is planned. +Manual adaptation guidelines are the only conversion assistance. + +## 1. Establish a small normative contract and baseline + +Files: `../miniexpr/doc/dsl-spec/1.0.md`, `artifact-1.0.md`, a machine-readable +feature/type/function matrix, and `../miniexpr/tests/portable-dsl/` fixtures. + +1. Retire unreleased 0.1 validation/loading and obsolete freeze documents. Retarget + useful encoding/numeric/masking/validation fixtures to draft 1.0, updating + independent references for operand typing and checked arithmetic. +2. Inventory actual builtin registrations and interpreter implementations once. + Classify each operation as included, existing full-DSL only, or excluded; + include arity, type combinations, result type, errors and accuracy policy. + Avoid another unbounded backend audit. Review omissions of useful existing + operations before freezing the matrix. +3. Cover arithmetic, comparisons, bitwise operations, shifts, casts, `where`, + control flow, supported math, fixed-width string operations, ND symbols and + the existing numeric/Boolean reduction families. Do not assume scalar and + reducing forms of similarly named functions share semantics. +4. Spell out floor division/remainder signs, integer power and shift overflow, + signed-zero behavior, NaNs, infinities, local type joins, short-circuiting and + evaluation order. Recommend typed locals with a statically determined join + type and rejection when no supported join exists; no dynamic dtype changes. +5. Specify loop caps and missing-return errors. Resource caps are host policy; + an error must not become a partial successful result. Preserve existing + configured caps rather than inventing a new execution sandbox. +6. Use semantic capability IDs for ND context, fixed strings and block reductions. + A declared capability is a requirement, not permission to skip validation. + +Exit: a finite matrix and example fixtures cover every admitted operation; +no undefined implementation behavior is advertised as portable semantics. + +## 2. Introduce versioned semantic compilation and conservative dispatch + +Primary files: native `dsl_portable.c`, `dsl_compile.c`, +`dsl_compile_internal.h`, `dsl_compile_support.c`, `dsl_artifact.c`. + +1. Thread a portable semantic profile through compilation and compiled program + metadata. Keep it out of process-global mutable switches. +2. Validate normalized source, resolved call targets, compiled node types, + effects, return cardinality and inferred capability requirements. Use both + source and typed-tree gates. +3. Reject callbacks, complex values, `print`, fast/contract FP, dynamic widths, + arbitrary memory access and unsupported reduction placements. A reduction + condition inside a loop is not a reduction assignment in the loop body. +4. Initially mark every 1.0 program interpreter-only. Optional JIT requests fall + back before compilation/execution of an incompatible backend. Explicit + required-JIT requests fail clearly when no certified path exists. +5. Preserve source locations through compilation and expose stable unsupported, + binding, format and evaluation error categories. + +Exit: 1.0 kernels cannot accidentally execute +under full-DSL output-context typing or an uncertified JIT. + +## 3. Implement checked operand-typed numeric interpretation + +Primary files: `dsl_compile.c`, `dsl_eval.c` and the shared native expression +typing/evaluation helpers they call. Locate shared helpers before changing them; +make changes profile-aware where ordinary DSL semantics differ. + +1. Infer every expression's computation dtype independently of declared output. + Preserve weak-literal metadata until contextual typing is complete. Insert + explicit typed conversion nodes rather than relying on output buffers. +2. Implement the approved promotion table and casts. Round float32 intermediates + at operation boundaries; prohibit implicit contraction/reassociation in the + strict profile. Test that changing output dtype does not change intermediates. +3. Implement checked integer add/subtract/multiply/negation, division/remainder, + shifts, powers and all narrowing/conversion paths actually admitted by the + matrix. Check before performing a C operation that could overflow or be UB. + Do not rely on signed overflow, invalid shifts or out-of-range C casts. +4. Check float-to-integer limits without rounded-boundary errors, including + int64/uint64 extrema. Fractional values truncate toward zero before range + acceptance is decided. Cover exact large integers without double round trips. +5. Apply numeric-zero/one Boolean arithmetic consistently to intermediates, + explicit casts and final output. Preserve fractional division before truth. +6. Propagate failures from nested expressions and final conversions. Only active + lanes execute checked operations. Short-circuit branches and `where` must not + report failures from unselected lanes; implement masked operand evaluation + rather than evaluating invalid discarded branches eagerly. +7. Document floating math domains and per-function tolerances. Use a bounded + cross-platform accuracy contract informed by existing libm implementations; + do not make bitwise transcendental equality a release requirement. +8. Keep workspace and error state per evaluation. On failure, native output is + unspecified and must not be consumed; Python raises rather than caching or + returning that block as a successful result. + +Exit: exact integer/error cases, Boolean fractions, float32 rounding, mixed +types and inactive invalid branches pass interpreter conformance on native and +WASM targets. Additional JIT arithmetic work is unnecessary for this milestone. + +## 4. Make masking and limited block reductions explicit + +Primary files: `dsl_eval.c`, `dsl_eval_internal.h`, `dsl_compile.c`, +`miniexpr_eval_reduce.c`, and ND evaluation adapters. + +1. Represent valid lanes and currently participating lanes explicitly. Track + branch, loop, break, continue and return masks; nested conditions reduce over + the mask participating at that program point. Continue excludes the rest of + that iteration but does not permanently remove a lane from the next one. +2. Top-level reductions see valid block elements; nested supported conditions see + active valid elements. Never include padded lanes or stale local storage. +3. Retain the restriction on reduction-valued assignments/returns in control-flow + bodies. Admit top-level reductions and supported conditions, including nested + `if all(...)` in Mandelbrot. Validate placements structurally, not by a string + search or a blanket ban on nonzero nesting depth. +4. Accumulate in logical C-order within a block; skip inactive lanes without + changing the order of remaining lanes. Disable parallel/tree reduction and + fast reduction shortcuts for this profile unless they give the required order. +5. Recommended operation details for the normative matrix: widen signed/Boolean + sum/prod to int64 and unsigned to uint64; keep floating sum/prod input dtype; + integer mean uses float64; floating mean uses its floating input dtype. + Checked integer accumulation applies at every step. Mean uses the specified + ordered sum followed by division by the participating count. +6. Use approved empty identities for any/all/sum; recommend prod(empty)=1 and + mean(empty)=NaN. Empty min/max errors. Recommend NaN propagation for numeric + reductions, deterministic signed-zero ties for min/max, and NaN-as-true for + truth reductions, consistently with Boolean conversion. Record these rules + explicitly and compare against existing behavior before implementation. +7. Distinguish reduced scalars from per-lane values in compiled metadata and + allocation. Broadcast scalar operands only onto participating lanes. + +Exit: Mandelbrot interpreter results survive changes in partition when its +early-exit optimization is semantically neutral; a deliberately block-dependent +example changes exactly as specified. Include partial masks, edge padding, +empty groups, continue/break/return, ordered cancellation and overflow cases. + +## 5. Extend native artifacts for context, widths and cardinality + +Primary files: `dsl_artifact.c`, `miniexpr_artifact.h`, `dsl_portable.c`. + +1. Keep standalone UTF-8 JSON, explicit schema/language versions, strict semantic + requirements, named signatures, exact typed constants and optional bindings. + Extend integer encodings for the admitted widths; retain decimal integer + strings, IEEE floating hex and Boolean values. +2. Add string descriptors with code-unit kind and fixed width. Recommend canonical + hex bytes for bytes constants and fixed-endian 32-bit code-unit hex for Unicode + constants, preserving values without assuming all slots are valid UTF-8 text. + Native buffers use host representation; adapters perform endian conversion. +3. Add result cardinality and queries for itemsize/output capacity. Introduce an + extended evaluation descriptor/API rather than silently changing the old + `me_artifact_eval()` ABI, whose output currently has `nitems` elements. +4. The extended call supplies logical shape, block origin/extent, valid-lane + description and explicit output capacity. Define `_flat_idx` in the logical + domain, `_i` as coordinates in that domain and `_n` as logical sizes. + Broadcasting inputs does not change the output coordinate domain. +5. Validate required context, dimensions, products, byte capacities, integer + conversions, ownership and overlap before evaluating. No array-sized constant + replication; no mutable context retained on a shared handle. +6. Distinguish zero-element elementwise calls from explicitly empty block-scalar + reduction calls; the latter must execute empty-group semantics. +7. Validate inferred capabilities against artifact declarations. Unknown versions + or unavailable capabilities fail explicitly, without Python reconstruction. + +Exit: a standalone C runner loads each capability fixture and evaluates supplied +buffers/context using only native code; incorrect descriptors fail deterministically. + +## 6. Admit existing fixed-width strings and logical ND kernels + +Native files: existing string implementation, `miniexpr_eval_dsl_nd.c`, +`miniexpr_eval_nd.c`, plus compile/validation and artifact descriptors above. + +1. Reuse native string operations and width inference. Specify first-NUL behavior, + full-width unterminated slots, no Unicode/bytes mixing, bytes comparison rules, + Unicode code-unit order, case-expansion truncation and static width bounds. +2. Pin the Unicode mapping/whitespace data semantics used by admitted operations; + avoid locale-dependent behavior. Separate arbitrary byte constants from the + current restriction on non-ASCII source literals used against bytes operands. +3. Check branch/local width joins and allocate the inferred return width, not a + NumPy approximation. Reject unbounded results before execution. Keep strings + interpreter-only; a host may pack fixed-width output into varlen storage. +4. Define sliced/view authoring domains explicitly: a newly constructed kernel + over a view has its declared logical domain; reading a slice of an existing + saved kernel retains that kernel's original coordinates and reduction groups. +5. Exercise zero-input constructors using ND context, sliced domains, broadcasting + and padded edge blocks. Pass context through native evaluation, not synthetic + Python callbacks or hidden global arrays. + +Exit: representative existing string/ND kernels export and execute identically +in C and Python, including widths and coordinates, with no JIT dependency. + +## 7. Python bridge, normalization and lazy execution + +Files: `src/blosc2/portable_kernel.py`, `dsl_kernel.py`, `blosc2_ext.pyx` and +associated native declarations, `lazyexpr.py`. + +1. Make export/validation APIs default to draft 1.0; reject retired versions. + select 1.0 explicitly for new persistence until the public-default transition + is documented. Import remains native-only. +2. Reuse NumPy-call, string-syntax and static-row normalization; record named + column bindings and snapshot supported typed constants, including strings. + Reject opaque captures without invoking arbitrary conversion hooks. +3. Pass context, width, shape/cardinality and evaluation errors through Cython. + Extend `PortableKernel.evaluate()` accordingly. Build interpreter-backed lazy + execution around the native handle, not around `kernel_from_source()` or exec. +4. Define a persisted logical block grid with shape, block extents, domain origin + and C-order traversal. Bind it once for reduction-bearing kernels; do not + derive it anew from current physical chunks at each evaluation. +5. Schedule complete original reduction groups for elementwise partial reads; + gather/scatter across physical storage as necessary. Elementwise-only kernels + may be tiled freely while retaining original logical ND coordinates. +6. Implement the approved block-scalar output mapping and distinguish input + domain shape from result shape. Validate immutable bound dtypes/widths and + domain requirements when reopening referenced operands. +7. Preserve author-written compiler pragmas as compiler selection preferences; + local defaults must not override them. They do not exempt the backend from + portable semantic checks or disable the agreed interpreter fallback. + +Exit: ND/string/reduction kernels run lazily through native handles, with slice +results and reduction grouping unchanged after storage rechunking. + +## 8. Enforce portable-only kernel persistence end to end + +Files: `b2objects.py`, `lazyexpr.py`, `deserialization.py`, relevant open/from-frame +entry points and CTable computed/generated-column serializers. + +1. Trace every DSL-kernel save/load route: structured B2Objects, older LazyUDF + metadata paths, embedded/nested lazy operands, CTable columns and frames. + Use one encoder/validator and one native decoder rather than divergent formats. +2. Preflight normalized kernel portability and typed bindings before creating or + replacing the destination. Save the artifact plus explicit operand references, + logical context/partition and container format discriminator. +3. Remove broad exception-driven fallback to source-based persistence for DSL + kernels. A portability failure must survive to the caller with location/reason. + Do not redefine ordinary non-kernel LazyExpr persistence by accident. +4. On load, distinguish portable, legacy and malformed records using metadata + only. Validate portable artifacts natively before binding/execution; never + reconstruct Python from a portable record, even after validation fails. +5. Apply explicit legacy opt-in consistently, including recursive operands. + Review current hard-coded `deserialize="full"` calls in `b2objects.py` and + effective defaults in `deserialization.py`; do not allow an internal reopen + to bypass the user's selected policy. +6. No automatic migration, conversion command or source-rewrite-on-open. + Document manual changes and explicit legacy use. Container/data-source + portability is separate from standalone kernel portability; demonstrate C + execution of the stored artifact with supplied bindings, without promising + C support for every Python remote/container protocol. + +Exit: all new DSL save routes reject excluded features; round trips preserve +native-only execution and block context; legacy files require opt-in everywhere. + +## 9. Optional JIT eligibility, not a JIT implementation project + +After interpreter milestones, build a conservative eligibility predicate over +the typed program and backend capabilities. Its default answer is no. + +- Allow only operation/type/context combinations already shown to implement 1.0 + semantics. Include semantic version, dtype/width and relevant policy in cache + identity so cached full-DSL programs cannot masquerade as 1.0 programs. +- Keep checked integers, unsupported casts, strings, ND/reduction combinations + and uncertain masking on the interpreter until individually certified. +- Optional JIT unavailability or unsupported lowering selects interpreter before + execution. Do not retry an executed, failing kernel using another backend; + arithmetic/domain failures remain failures. +- Required-JIT remains an explicit host diagnostic mode: report unsupported + acceleration instead of falsely claiming JIT execution. +- Retain the Windows interpreter baseline and WASM exact-integer fallback. + No target is required to acquire a new code generator for 1.0. + +Exit: eligible kernels match interpreter semantics; ineligible kernels execute +correctly by default and report a reason when JIT is explicitly required. + +## 10. Bounded conformance and integration verification + +Do not cross-product dtype, operation, backend, length and storage dimensions. +Put the detailed semantic corpus in miniexpr and keep approximately 100–200 +orthogonal Python portable tests, restructuring existing cases as necessary. + +Native fixtures cover exact boundary/error cases for each checked operation; +representative promotion/literal pairs; rounding and inactive-lane behavior; +Mandelbrot and masked reductions; ND origins/padding; string encodings/widths; +artifact errors, capacity and ownership; immutable concurrent handles. + +Python integration covers normalization and captures, standalone-native import, +native error propagation, each save route, legacy opt-in, rejection without +partial destination replacement, partial reads/rechunking, scalar-block shapes, +default fallback and explicit required-JIT failure. Use exact expected values +for integers rather than a NumPy overflow reference. Floating expectations use +the specified rounding/order or function-specific tolerance. + +Normal Python tests use the published/pinned miniexpr and self-contained fixtures. +Keep external corpus/C-runner checks opt-in through the established environment +variables; never infer a sibling checkout at runtime. + +Run targeted tests after each milestone, native sanitizers for masks/capacity/ +arithmetic, then full native/Python CI including Windows and WASM at integration. +Use the `blosc2` conda environment for Python, builds and tests. Smoke-test an +isolated wheel built without local native overrides. Test artifact-disabled +builds report missing capability cleanly. Update the published native pin only +after the required native code is available; no unpublished-checkout dependency. + +## Delivery order and completion checklist + +### Autonomous execution and durable handoff + +Follow the milestones in order, recording progress and verification after each. +The autonomous implementation target is completion of the beta-readiness gates, +not collection of external beta feedback or automatic freezing of the contract. + +- Repository scope covers the necessary changes in Python-Blosc2 and miniexpr. + Follow each repository's local instructions. Preserve unrelated user work; + avoid unrelated refactors and whole-language JIT audits. +- Prioritize interpreter correctness and use pre-execution JIT fallback. Do not + spend a milestone's effort implementing optional acceleration. +- Continue through routine implementation, build and test failures without + asking permission. Resolve ordinary details from approved semantics, verified + implementation behavior and the normative matrix. +- Pause only for unresolved semantic decisions, scope contradictions or external + blockers such as unavailable credentials. Record the precise blocker and + proposed resolution; continue independent unblocked milestones where possible. +- Commits, pushes, publishing and other repository/release state changes require + explicit user authorization consistent with repository instructions. This plan + does not itself grant those permissions. Run available local verification; + record any CI checks blocked by missing authorization or credentials. +- Keep a durable handoff in `plans/portable-DSL-1.0-progress.md`. Create it at the + start of implementation and update it after every milestone and before any + interruption or context handoff. Include completed checklist items, changed + files in both repositories, exact verification commands/results, unresolved + failures, decisions and their approval status, and the next concrete action. + Record relevant native revision/build/pin information so another session can + reproduce the environment. Do not store credentials or secrets. +- On resume, read this plan, the scope agreement, the progress file and current + working-tree changes before editing. Do not assume an interrupted command + completed or discard partial work. Reverify uncertain results. +- At completion, summarize delivered behavior and remaining external release + gates. Do not claim CI success, beta feedback or release publication without + evidence. + +Checklist status is **local implementation**, not release/platform certification; +authoritative commands/results are in `portable-DSL-1.0-progress.md`. + +1. [x] Finish finite normative draft type/arity/domain/tie/NaN/reduction/Unicode + matrix (Q1–Q3 confirmed), with analytic/exact-reference fixtures. Independent + function-specific cross-platform libm accuracy bounds remain a release gate. +2. [x] Add versioned compilation/interpreter-only 1.0 dispatch, including public + core-only source validation and extended ND/cardinality/string-width descriptor. +3. [x] Complete checked operand-typed numeric semantics and active-lane errors. +4. [x] Complete reduction masks/cardinality, including rejection of divergent + scalar return sites and preservation of finished-lane iterator values. +5. [x] Extend artifact ABI/schema/context and native runner/descriptor fixtures. +6. [x] Admit bounded fixed strings and logical ND descriptors through native code. +7. [x] Integrate Python native handles and immutable logical-block scheduling. +8. [x] Enforce validated saves/native loads and recursive explicit legacy opt-in + across the implemented disk/frame/container/table routes, with safe MessagePack + defaults. No source fallback; table row-domain modes are explicit. + New authored DSLKernel-backed LazyUDF saves automatically normalize and export + validated 1.0 artifacts, with typed scalar/capture snapshots and named array + bindings. Preserve the elementwise domain and current uniform logical block + grid; reject true block-scalar kernels with explicit portable API guidance. + This is authoring preflight, not automatic migration of loaded legacy carriers. + Explicit independent-row table APIs accept DSLKernel authoring plus output + dtype/cardinality directly; old vector-column APIs have no row-domain contract + and reject persistence with a specific ambiguity diagnostic. +9. [x] Conservative interpreter-only dispatch and pre-execution optional JIT + fallback; required-JIT rejects unsupported acceleration explicitly. +10. [x] Complete release verification gates for the tested platform matrix: + - [x] Compact local Python/native conformance and full local ASAN/UBSAN. + - [x] Isolated native-development override wheels and artifact-disabled smoke. + - [x] Draft/API docs build; changed portable page free of diagnostics. + - [x] Native Windows/Linux/macOS/WASM conformance matrix (all seven jobs green). + - [x] Independent finite math corpus and stock-pin macOS execution: 284 samples, + 40 canonical operations, both floating precisions, function-specific budgets. + - [x] Execute the independent math corpus on remaining Python release platforms + (no universal ULP bound inferred from the finite corpus or host libm). + - [x] Publish/pin the native revision and verify a stock-pin macOS ARM64 wheel. + - [x] Complete the remaining stock-pin platform wheel matrix (27 successful jobs; + retain the existing untested macOS Intel cp315/cp315t dependency exception). + - [x] Whole-tree docs disposition: fix parsing errors, exclude generated input + trees, and inventory/defer existing image/reference warnings outside Menudet. + +Each step must land as a usable tested increment; reduction and string fixtures +can be prepared while numeric interpretation is implemented, without introducing +unapproved semantics. If a useful existing feature cannot fit the finite matrix, +raise the specific incompatibility rather than silently shrinking scope. + +Release documentation includes normative native specs, Python API/persistence +guidelines, manual legacy adaptation, standalone C examples, and corrected DSL +reference statements. These checks establish readiness for the beta milestone +below; freezing the public 1.0 contract additionally requires beta feedback. +Do not tie completion to broad JIT acceleration, migration utilities or new +remote-storage protocols. + +## Release milestone: miniexpr beta and portable 1.0 release candidate + +Once the implementation checklist passes, move miniexpr into beta with portable +DSL 1.0 as a **release-candidate specification**. Interpreter conformance across +supported platforms, native artifact loading, and working persistence integration +are the entry gates. Limited JIT coverage does not block beta: correct +pre-execution interpreter fallback is sufficient. + +Keep the two release promises distinct: + +- **Miniexpr beta:** implementation and APIs are approaching release readiness; + broader use should exercise integration, usability and performance. +- **Portable DSL 1.0, once frozen:** a public compatibility commitment independent + of whether the miniexpr implementation is still in beta. + +During beta, exercise real persisted kernels and cross-language C/Python loading, +including ND context, fixed-width strings, block reductions, partial reads and +legacy opt-in. Resolve contract ambiguities and compatibility issues found in +that feedback before freezing 1.0. Clearly label release-candidate artifacts and +their compatibility status; do not present draft artifacts as already covered +by the final 1.0 guarantee. + +- [ ] Enter miniexpr beta after the implementation and verification gates pass. +- [ ] Publish portable 1.0 as a release-candidate specification during beta. +- [ ] Review real-world persistence and cross-language feedback; resolve blockers. +- [ ] Freeze and publish the portable 1.0 compatibility contract after that review. + +Further JIT acceleration remains an independent optimization, not a prerequisite +for either beta entry or contract freeze. diff --git a/plans/portable-DSL-1.0-progress.md b/plans/portable-DSL-1.0-progress.md new file mode 100644 index 000000000..42a46092f --- /dev/null +++ b/plans/portable-DSL-1.0-progress.md @@ -0,0 +1,1352 @@ +# Portable DSL 1.0 implementation progress + +## Table API and LazyExpr compatibility follow-up + +The user approved unifying table kernel registration and restoring ordinary +LazyExpr persistence, then explicitly requested **no `add_portable_*` aliases**. +The final policy retains trusted legacy execution through explicit +`deserialize="full"`; it supersedes the briefly proposed unconditional rejection. + +- `add_computed_column()` / `add_generated_column()` now export and validate + authored kernels and accept native artifacts with named bindings. Positional + lists and column-bound DSL LazyUDFs adopt the same independent-row contract. + `assign()` uses the same normalization; new registrations have no vector-column + fallback. String/lazy expressions and non-kernel row transformers are unchanged. +- Removed the separate portable-column methods and public `row_domain` argument. + New metadata omits `row_domain`; prior draft metadata with the independent + marker still validates, and unknown grouping markers reject. Fixed-shape row + reductions use `cardinality="block_scalar"`. +- Safe loading rejects legacy table-column recipes before source reconstruction, + with an explicit message naming block/batch-dependent behavior and the trusted + `"full"` opt-in. Full loading preserves legacy execution, not reinterpretation. +- Removed the new arithmetic-only LazyExpr AST restriction. Existing expression + validation remains. Ordinary numerical-expression round-trip tests now reopen + with the default policy; nested legacy/proxy/other full-only payloads stay gated. +- Added default round trips for `sqrt(x)`, `sum(x)`, `x.mean()` and `x[0]`, row-local + reduction tests, persisted legacy computed/generated opt-in tests, and artifact + registration/no-mutation checks. Updated API docs, release notes and example. + +Verification and final CI run links are recorded below as they complete. Earlier +sections describe the original API and are historical where they differ here. + +Local verification: the full default suite (including module doctests) passed +**11,291 cases**, with 55 existing skips. All 284 independent math samples passed, the Menudet +example ran, and Ruff/format/whitespace checks passed. Sphinx completed with no +ERROR diagnostics or warnings on the modified Menudet/persistence pages; the +previous unrelated whole-tree warnings remain outside this change. + +## Published release certification — green platform matrix + +The user authorized publishing changes and monitoring/fixing CI until green. +Native Menudet documentation was published as miniexpr +`36770f87e10f1b92bfe0f9f92a7039eba2e976ea`; all seven native jobs passed in +[37618199852](https://github.com/Blosc/miniexpr/actions/runs/37618199852). +Python still pins the already-green runtime revision `3418cdce...`: the subsequent +native changes are documentation/matrix metadata only, not runtime changes. + +Python release-hardening and certification fixes are published on `portable-dsls`. +Certified implementation/workflow revision: +`e768bc538cb096c7f5c669046ed3aed5e55d9c6b`. Successful runs: + +- [Tests 37622410469](https://github.com/Blosc/python-blosc2/actions/runs/37622410469) + — all standard platform jobs green, including independent math reports. +- [WASM 37622410514](https://github.com/Blosc/python-blosc2/actions/runs/37622410514) + — Python/Pyodide suite plus the independent corpus green. +- [Python wheels 37622420247](https://github.com/Blosc/python-blosc2/actions/runs/37622420247) + — **27 successful jobs**: seven native/WASM build jobs and twenty abi3 test + jobs across Python 3.11–3.15. Includes Linux x64/ARM64, Windows x64/ARM64, + macOS x64/ARM64, Pyodide 3.13/3.14 and free-threaded builds. PyPI upload is + intentionally skipped (no tag/publication). The pre-existing skip of testing + macOS Intel cp315/cp315t wheels is retained for unavailable pre-release dependency + wheels; do not claim those specific wheels were tested. + +The first two wheel runs (`37618308296`, `37620050340`) failed because importing +the extension safely enables the GIL on free-threaded Python, emitting a +RuntimeWarning that pytest promoted to an error. A command-line filter initially +failed because pytest treats CLI warning messages literally, not as regexes. +Moved the **exact extension-specific regex** into `pytest.ini`; other runtime +warnings remain errors. A local synthetic check confirms both acceptance of the +expected import warning and rejection of an unrelated numeric RuntimeWarning. +No `PYTHON_GIL=0`, no no-GIL safety declaration, no tolerance changes or new +conformance skips were introduced. + +Added `workflow_dispatch` to the wheel workflow to certify development revisions +without release tags. Both wheel Pyodide versions and each abi3 matrix job now +also run the independent corpus. + +Downloaded and verified all five native Python math reports: every report has +**284 samples and zero failures**. Maximum observed ULP: Windows **2**, Linux +**2** (including NumPy 1.26 and Python 3.14 jobs), macOS ARM64 **3**. Reports are +under `/menudet-native-ci-math-reports`. Wheel/WASM corpus executions +are additionally certified by the successful corresponding job commands. + +The earlier external-platform/published-branch blockers below are superseded. +Release certification for the tested matrix is now green; final release version, +publication, beta feedback and compatibility freeze remain separate decisions. + +## Menudet / 4.15.0 release hardening — current handoff + +The user selected **Menudet — a little language for portable computation on +arrays and tables**, and approved the release-hardening plan. Menudet is the +portable language; miniexpr implements it. Existing API/format identifiers remain +unchanged. Broader JIT acceleration is deliberately separate: the native guard +still forces portable interpretation, preserving checked/ordered semantics. + +### Delivered in this increment + +- Both version fields now read `4.15.0.dev0` (development, not final publication). +- Python reference, release notes and native spec introductions name Menudet, + retain the draft warning, and distinguish full DSL from portable semantics. +- `examples/menudet.py` demonstrates the same array/table kernel plus explicit + logical block-scalar reductions; assertions passed against the installed wheel. +- Added `scripts/certify_menudet_math.py` and its checked-in JSON corpus: **284 + samples, 40 canonical functions, float32 and float64**, generated with mpmath + 1.3.0 at 100/200 decimal digits. Regeneration equality and deliberate failure + detection passed. No new runtime dependency; mpmath is needed only to regenerate. +- Function-specific finite sample budgets and coverage limitations are documented + in `doc/reference/menudet_accuracy.md`. These are not universal ULP guarantees + or certification of all possible inputs. Native exceptional/exact/alias fixtures + remain required. +- Wired the corpus into native Python, WASM and installed-wheel CI. Native jobs + upload platform reports; tested wheels also execute the seven portable modules. + +### Verification + +All Python/build commands used conda `blosc2`. Fresh stock build fetched native +`3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4`; both source override cache entries +are empty. Isolated imports asserted the installed target path and excluded the +editable finder only within the test subprocess. + +- Working-tree stock wheel: **11160 tests passed, 53 skips** (excluding heavy, + network and TUI), plus **120 installed-module doctests passed, 2 skips** in a + clean working directory. The serial combined invocation initially encountered + one relative-filename save failure: installed doctest modules are outside the + repository root fixture's isolation. All tests passed, and all doctests passed + on the clean-directory rerun; no user's existing file was removed to fix it. +- Independent corpus on that wheel: **284 passed**, maximum observed error + **3 ULP** (lgamma/float64); no tolerance changed after observing results. +- Built an **8,044,023-byte clean sdist**, verified inclusion of checker, corpus + and example, built a wheel from the unpacked sdist, and isolated-installed it. + This wheel passed **183 portable tests, 17 explicit external opt-in skips**, + plus all **284 independent math samples**. +- Clean-sdist wheel SHA256: + `c7fcc99d44fe12a7cdffc717eff4bc04450954c6d51eb98b282d09c2646b0d67`. +- Clean snapshot copied only tracked files and the four requested new files; + unrelated untracked local work was neither packaged nor deleted. A direct + dirty-checkout sdist attempt timed out and is not a release artifact. +- Ruff, format, workflow YAML, reference regeneration and whitespace checks pass. + +### Documentation disposition + +Fixed installation list-table indentation, stopped treating generated `doc/html` +and autosummary stash trees as sources, and documented NumPy's finfo alias without +reparsing its incompatible nested sections. Documented deserialization policy +types/errors and taught the public-API tripwire to recognize manual `py:class`. +Fresh whole-tree Sphinx now has **zero ERROR diagnostics**, and no Menudet page +diagnostic. It still has existing non-Menudet warnings: the fresh build reported +769 (639 autosummary stub, 100 duplicate object/label, 11 ambiguous references, +2 image, 17 other); the final follow-up with notebook execution disabled reports +772, with no undocumented-finfo warning. They remain visible, unsuppressed +documentation backlog, not a claim of a warnings-as-errors clean build. Menudet +pages and parser errors are the scope of this draft's documentation acceptance; +broad autosummary/reference cleanup is explicitly deferred. + +### External gates still pending + +The updated Python branch/workflows are not remotely available, so no new Python +platform job can certify these changes yet. Native miniexpr's seven-job matrix +is already green, but it does not certify Python artifacts/wheels. Windows, +Linux, Pyodide, free-threaded/ABI/platform wheel jobs must run the updated corpus +and integration tests and resolve any failures before releasing. No release or +beta publication occurred; final version metadata waits for those gates. + +Temporary directories under the approved temp root: `menudet-release-build`, +`menudet-release-wheel`, `menudet-release-installed`, `menudet-release-snapshot`, +`menudet-clean-sdist`, `menudet-sdist-unpacked`, `menudet-sdist-wheel`, +`menudet-sdist-installed`, `menudet-release-docs-clean`, and +`menudet-release-doctrees-clean`. Reports: `menudet-math-macos.json` and +`menudet-sdist-math-macos.json`. + +## Dependency pin updated after green native CI + +At the user's request, `CMakeLists.txt` now pins the green native revision +`3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4` from CI run `37612064807`. +Earlier notes saying the Python pin remains at `55c882b` are superseded. +Stock Python wheels have not yet been rebuilt/tested against this updated pin. + +## Native CI green (supersedes investigation below) + +At the user's request, published the fixes and monitored/repaired CI until green. +Revision `02791f338a853af517b6fedbd38c18d5a2b8da1a` fixed WASM and Windows +builtin recognition, but run `37611459450` exposed a second Windows issue: +identical-code folding merged the floating cast callback with real/conj identity +callbacks, misclassifying their output type. A unique volatile read in the cast +callback preserves its address identity without altering signed zero or NaNs. +The anchor fixture now reports the source and dtypes on a type mismatch. + +Final published revision: `3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4`. +CI run [37612064807](https://github.com/Blosc/miniexpr/actions/runs/37612064807) +completed **successfully: all seven jobs green**, including Windows x64/ARM64, +Linux x64, Linux ARM64 interpreter/JIT, macOS, and WASM (including side-module +helper and JIT trace checks). Targeted local portable interpreter test also passed +after the cast-identity repair. Python's dependency pin remains at `55c882b`; +this entry does not claim Python stock-wheel certification of the final fixes. + +## Native CI failure investigation (2026-10-07) + +CI run `37610312891` on native revision `55c882b` failed on Windows x64, +Windows ARM64, and WASM. Both Windows failures occur while compiling the portable +`hypot(x, 2.0)` fixture: libm address recognition fails. WASM compilation fails +because its fenv lacks `FE_INVALID`; the new test also incorrectly assumed host +rounding/thread facilities. + +Local fixes in miniexpr: + +- `src/functions.c`: typed builtin recognition also matches the exact builtin + registration address, handling CRT inline/import aliases such as Windows hypot. +- `tests/test_dsl_portable_interp.c`: WASM retains numeric/literal/environment + checks under its fixed nearest rounding, without requiring unavailable exception + flags or pthreads. Native upward-rounding, exception and concurrency checks stay. + +Verification: complete native interpreter suite **325 passed**; local Emscripten +interpreter-only suite **48 passed**; CI-style WASM with JIT enabled **50 passed**; +targeted native portable interpreter ASAN/UBSAN **passed**. Existing duplicate +library and warning-enabled unused-function warnings remain; no WASM build warning +was observed. Remote Windows jobs still require revalidation with these fixes; +the reported failed CI run is not claimed green. Published pin is unchanged. + +Build directories: `/miniexpr-portable-1-interpreter`, +`/miniexpr-portable-1-wasm-fix`, and +`/miniexpr-portable-1-sanitized`. WASM was configured with `emcmake`, +Node as emulator, SLEEF OFF, then TCC JIT ON/bundled host TCC OFF/trace ON to match +the CI configuration. All commands used `conda run -n blosc2`. + +## Published dependency update — latest status + +The user authorized committing/pushing miniexpr and committing the Python +implementation plus dependency pin. Native implementation is published on +`Blosc/miniexpr` main at `55c882b238c4daf0523a52a44d5d84db56af24e4`. +Python-Blosc2 `CMakeLists.txt` pins that exact revision. Earlier notes saying the +published pin is unchanged are historical and superseded by this update. + +A fresh stock wheel (no local source overrides) built successfully using that +revision. Both `FETCHCONTENT_SOURCE_DIR_MINIEXPR` and +`FETCHCONTENT_SOURCE_DIR_BLOSC2` are empty in its CMake cache, and the fetched +native checkout reports the pinned SHA. The isolated installed wheel passed all +seven portable test modules: **183 passed, 17 external opt-in skips** (200 cases). +The editable development installation was not replaced. + +Verification commands used `conda run -n blosc2 python -m pip wheel . --no-deps +--no-build-isolation --wheel-dir /portable-1-published-pin-wheel +--config-settings=build-dir=/portable-1-published-pin-build`, followed +by a separate `pip install --no-deps --target /portable-1-published-pin-smoke` +and pytest with the editable finder excluded only in that subprocess. Wheel SHA256: +`434b4cb6fdb1e5fa11dd088a749a42359ed798242e8de05cef69710d642c9d87`. + +This verifies the published-pin macOS ARM64 wheel, not Linux/Windows/WASM +certification or beta readiness. Unrelated user work remains unstaged. + +## CURRENT STATUS — authoritative handoff (supersedes all older sections below) + +### Unreleased portable 0.1 retired; draft 1.0 is the public default (2026-10-07) + +The user's corrected release scope is implemented in both repositories. Portable +0.1 was never released and has no compatibility obligation. The public story is +experimental/draft portable 1.0 versus legacy Python-specific persistence, whose +active reconstruction requires explicit `deserialize='full'`. No automatic +migration or reinterpretation occurs. Ordinary full-DSL semantics are unchanged. + +- `DSLKernel.export`, its authoring helper and `validate_portable_dsl` default to + 1.0. Removed the Python 0.1 dtype/call allowlists, scalar encoder and export + branch. Old native builds report unavailable draft support instead of exposing + their retired loader; the published native pin remains unchanged. +- Native `dsl_portable.c` now contains only typed draft validation. The artifact + loader rejects unsupported schema/language versions before interpreting their + envelope as 1.0, then strictly validates context/signature/capabilities/constants. + Removed the old source filter, full-profile artifact compilation, tiled evaluator + state and mask/ND rejection branch. `me_artifact_eval` remains a useful rank-zero + numeric elementwise call adapter to descriptor evaluation, not format support. +- Portable compilation no longer inherits full-DSL `ME_DSL_FP_MODE` defaults; + it uses the strict parsed draft default and still rejects explicit non-strict + source. Ordinary full-DSL environment policy remains intact. +- Removed obsolete freeze/language/artifact draft documents and the 18-entry + frozen-excluded manifest/category/rejection tests. Retargeted the native corpus + and runners to interpreter-first 1.0. Separate full-DSL audit cases remain. + Five corpus reference files now reflect operand-driven float32 arithmetic, + explicit float64 casts and integral Boolean-negation zero, rather than output + context typing. Native encoding, schema, ownership, binding and descriptor + regressions remain and now load 1.0. +- The Python host suite retargets the former frozen boundary to actual arithmetic + admission/results and checked overflow, preserving all 200 orthogonal cases. + Mixed captures and NumPy normalization now execute; the old rejected string + capture case now verifies a typed Unicode snapshot. Optional JIT requests always + assert interpreter fallback, never pretend `has_jit=True`. External corpus/runner + tests use explicit environment paths only and now exercise portable artifacts. +- User-facing Python/native API docs and specifications describe draft 1.0 only, + with a prominent experimental notice and no invented beta/platform claim. + Implementation plan/scope current decisions record retirement. Historical + verification below and unrelated user files/logs are retained. + +Verification so far, all commands in conda `blosc2`: + +- Complete native CTest: **325 passed**; complete ASAN+UBSAN CTest: **325 passed**. + The count drops from 343 solely because the 18 obsolete frozen rejection gates + were removed; useful arithmetic/masking/encoding/validation fixtures remain. +- Seven portable Python modules: **183 passed, 17 external skips = 200 cases**. + Explicit external native corpus/runner integration: **176 passed, zero skips** + across `test_portable_artifact.py` and `test_dsl_portable.py`. +- Full default Python suite: **11280 passed, 55 skipped**, 36.35s, after the final + bridge/validator changes and retained Unicode capture regression. +- Native warning-enabled rebuild retains existing unused helpers/aggregate + initializers/unknown SIMD pragmas and duplicate-library linker warnings. No new + warning in the retirement validator/artifact/runner changes or sanitizer finding. + +Remaining release gates are unchanged: independent function/platform accuracy, +Windows/Linux/WASM and stock-published-pin wheel certification, authorized native +publication/pin update, and clean whole-tree documentation. Local development +override wheel results are not stock-published-pin evidence. + +### Final retirement verification and file handoff + +- Rebuilt editable extension with explicit local miniexpr and C-Blosc2 overrides. + After isolated wheel installs, rechecked source import path and active descriptor + support; the editable install remains in use. Published pin/dependencies unchanged. +- Isolated local-native-override wheel: all seven portable modules **183 passed, + 17 external skips**, 1.80s. Smoke subprocess removed only its own editable finder, + asserted the imported package was beneath its isolated target and used `-n 0`. + Wheel SHA256 `16a514a8063601c4a4eb1ce1ebfe1a0b13e1421c8e4cbb1e32126424127c290f`. + This is not stock-published-pin, platform or published-wheel certification. +- Separate artifact-disabled local-override wheel built/installed only to another + isolated target. Ordinary array arithmetic and draft numeric/string/ND source + validation pass; retired source version rejects and artifact loading explicitly + reports `NotImplementedError`. No installed editable replacement. +- Final native and ASAN+UBSAN suites: **325/325** each, 9.08s / 9.23s. Final + incremental builds report only the existing duplicate `-lm`/static-library + linker warning; no sanitizer error. Earlier warning-enabled full rebuild retains + the pre-existing warnings documented above, not a warning-free whole-tree claim. +- Ruff lint and format checks pass on all four Python files changed by retirement; + Cython compiled successfully in editable and both wheel builds. Both repositories + pass `git diff --check`. +- Sphinx exits successfully; neither changed portable/native-syntax Python page has + a diagnostic. Whole-tree output contains **931 WARNING / 4 ERROR lines**, including + existing installation list-table, NumPy/reference/image/generated-stub issues. + This remains an incomplete clean docs gate, not a release acceptance claim. +- Final callsite/version audit over source/tests/docs/examples/build/CI files finds + no retired portable support/default/freeze links; explicit 0.1 rejection tests + and numeric literals remain. Moved the prior numeric audit to + `../miniexpr/plans/numeric-audit-0.1-history.md` as a factual archive, out of public + specifications. Unrelated untracked user work and user logs are untouched. +- Final additive regression verifies an actual retired envelope (no `context`, + `core-scalar`, `scalar-per-element`) rejects specifically as unsupported version, + not as malformed 1.0. The containing diagnostics case reran **1 passed**; + lint/format and both whitespace checks passed again. No extra test-count grid. + +Retirement files (in addition to preserved earlier 1.0 implementation): + +- Python: `src/blosc2/dsl_kernel.py`, `portable_kernel.py`, `blosc2_ext.pyx`, + `dsl_artifact_bridge.h`; `tests/test_portable_artifact.py`, `test_dsl_portable.py`; + `doc/reference/portable_dsl.rst`, `dsl_syntax.md`; current plan, scope and progress. +- Native: `src/dsl_portable.c`, `dsl_artifact.c`, `dsl_compile.c`, `miniexpr.h`, + `miniexpr_artifact.h`, `dsl_portable_types.h`, `functions.c`; + `tests/CMakeLists.txt`, `test_dsl_portable_validation.c`, `test_dsl_artifact.c`; + both portable runner/README directories, affine JSON and five arithmetic/Boolean + reference fixtures; draft 1.0 specs and native DSL syntax/usage docs. + Removed `doc/dsl-spec/{0.1,artifact-0.1,draft-history-0.1}.md` and + `tests/portable-dsl/frozen-excluded.txt`; preserved historical numeric audit as above. +- Reproducible logs/builds/wheels are under the approved temporary root with + `retire-` prefix: editable, full/target/external Python, native/sanitized builds + and tests, wheel build/install/tests, disabled build/install/smoke and docs. + +## PRIOR IMPLEMENTATION HANDOFF — historical; superseded by retirement above + +Final parent-session check: the seven portable Python modules passed again +(**183 passed, 17 external skips**, 200 cases); whitespace checks passed in both +repositories. Local implementation milestones 1–9 are complete as recorded in +the implementation checklist. Remaining milestone 10 gates are external release +certification and pre-existing whole-tree documentation issues, not a claim that +miniexpr has entered beta or portable 1.0 is frozen. + +- Parent scope correction implemented: compliant newly authored DSLKernel-backed + LazyUDF saves/frames/stores/MessagePack/nested references now automatically + normalize and export validated native 1.0 recipes. Blanket rejection was too + strict and is removed. Shared `portable_from_lazyudf` preflight preserves typed + captures/scalar bindings, parameter-named array refs, output domain and existing + uniform logical block partitions. Block-scalar return/elementwise shape conflicts + reject with explicit `PortableKernel.lazy` guidance; chunk boundaries that split + that uniform block grid reject rather than inventing grouping. Loaded legacy + recipes are marked and remain nonmigrating, explicitly full-policy-only on load. +- CTable independent-row APIs accept DSLKernel authors directly with explicit + output dtype and reduction cardinality; authoring validates before registration. + Old vector-column APIs remain ambiguous about row-domain/grouping and now report + this specific reason and the usable authored independent-row route. Restored + add-square/loop/where/object/table roundtrips replace obsolete blanket-rejection + tests. Unsupported effect/callback and legacy saves preserve destinations. + Latest scope-correction verification: **11280 Python / 343 native / 343 + ASAN+UBSAN**, isolated development wheel **458 passed, 17 external skips**. + The compact portable boundary is unchanged at **200 cases**. Details below. +- Public validation closure is now implemented: core `me_validate_portable_dsl` + explicitly dispatches 1.0; new `me_validate_portable_dsl_ex` carries exact string + output width, logical ND rank and asserted/inferred cardinality. It uses the + existing profile parser/compiler with JIT OFF and no input buffers/callbacks, + execution, Python reconstruction or optional yyjson/artifact dependency. + Python `validate_portable_dsl(..., language_version='1.0', ndim=..., + cardinality=...)` routes to that API, preserving frozen 0.1 and reporting native + status/source locations. Actual local editable extension was rebuilt. +- The finite 1.0 matrix/spec now has normative draft arities/types, checked + combinatoric domains/width limits, ~ semantics, rounding/ties, NaN/zero rules, + ordered/empty/coherent reductions, Unicode 15.0.0 case data/exact whitespace + sets and string index/width rules. Independent analytic/hexadecimal references, + adjacent float steps, exact fused cancellation, all integer factorial limits, + canonical/alias math and explicit exceptional-value fixtures supplement—not + replace—the older same-host libm regression matrix. No universal accuracy + bound or external platform certification is invented. Implementation checklist + now distinguishes completed local milestones 1–9 from specific release gates. +- Independent correctness review found and fixed two boundary bugs (finished-loop + lane iterator clobbering and NumPy signed-minimum capture conversion warnings). + Post-review suites still pass: **11280 Python / 343 native / 343 ASAN+UBSAN**; + portable boundary remains **200 cases (183 passed, 17 external skips)**. + Both surfaced contract concerns are now **resolved**: typed structural validation + rejects divergent block-scalar returns as ambiguous output, preserving coherent + scalar control flow; MessagePack defaults to safe with explicit recursive caller + policy propagation. Post-fix isolated native-override wheel also passes the + entire 200-case portable boundary. See contract-follow-up addendum below. +- Installed conda `blosc2` editable extension uses the explicit local miniexpr + development override. Published native pin is unchanged. Local 1.0 tests run, + not skip. No commits, publishing or external platform certification performed. +- Native typed interpreter, draft schema/signature/constants, fixed strings, + ND descriptors, masked reductions, logical lazy partitions, partial reads, + streaming output/rechunking and safe portable persistence are connected. +- Added explicit independent-row CTable native computed/generated APIs: + `add_portable_computed_column` and `add_portable_generated_column`, requiring + `row_domain='independent'`. Each row is an original one-lane group; ND shape + (1,), origin (0,) are row-local, not global table indices. Append, refresh, + deletion and compacted save do not change these semantics. Fixed-shape ndarray + row inputs now also support scalar native reductions, with scalar/array input + broadcasting inside the complete row group, persisted immutable row shapes and + zero row-local ND origin. Table records carry artifact JSON, bindings and + row-domain metadata and load natively, never Python functions. Multi-row table + partitions/vector-valued output modes are explicitly unsupported; the native + standalone/lazy APIs still support the full logical-partition contract. +- Added explicit `DSLKernel.export(..., version='1.0', cardinality=..., ndim=...)`. + Reuses existing static row/string/NumPy call normalization and capture token + parameterization; typed signed/unsigned/float/byte/Unicode snapshots. Native + load validates normalized source and output widths/cardinality before export + succeeds. Default export remains frozen 0.1; no migration on import. +- Full default Python suite after resolving persistence regressions and a real + carrierless RemoteArray policy bug: **11280 passed, 55 skipped**. The policy + helper now handles objects whose `schunk` is None without losing effective + recursive policy. Previously failing old new-save expectations now verify + rejection of noncompliant kernels before destination writes; compliant authored + save roundtrips were restored by the parent scope correction above. Old + explicit-full load fixtures remain. +- Local native-development wheel built and isolated import/native reduction smoke + passed, explicitly **not a stock published-pin wheel result**. Artifact-disabled + development wheel built and isolated ordinary arithmetic/missing-loader smoke + passed. Editable install was not replaced by either isolated wheel install. +- Added finite native float32/float64 operation result-type/domain fixtures for + 26 numeric libm operations and six exceptional domains. These compare host + libm within explicit 8-epsilon relative bounds; independent cross-platform ULP + certification is still an external release-matrix gate, not claimed complete. + +Current changed inventory (including preserved earlier edits): native sources/tests +and draft specs listed in historical inventory below; Python bridge +`blosc2_ext.pyx`, `dsl_artifact_bridge.h`, `portable_kernel.py`, `portable_lazy.py`, +`dsl_kernel.py`, `ctable.py`, `deserialization.py`, `b2objects.py`, `lazyexpr.py`, +`schunk.py`, `core.py`, `ndarray.py`, `msgpack_utils.py`, `embed_store.py`, +`dict_store.py`, `tree_store.py`; `doc/reference/portable_dsl.rst`; portable tests, +legacy policy/persistence tests and affected table/ndarray/objectarray tests. +Most recent native closure: `src/dsl_portable.c`, `src/miniexpr.h`, and the +public-validation/independent-reference test functions. Keep unrelated user +files and all existing uncommitted changes. + +Table-copy/materialization and native signature/shape/dependency revalidation on +save and load are now implemented. Executed fixtures include static row/NumPy +normalization, typed numeric/Unicode snapshots, scalar/ND constructors, scalar +and vector-row append/extend/refresh, fixed-string row outputs, compact-copy, +deletion, materialization, disk/frame round trips, no Python reconstruction, and +malformed-artifact rejection preserving an existing destination. The portable +Python boundary remains **200 cases: 183 passed, 17 opt-in external skips**. +Native default suite **343/343 passed** and complete ASAN/UBSAN suite +**343/343 passed**; the two targeted artifact/interpreter sanitizer tests also +passed separately. Added descriptor initializers produce no new test warnings; +the full sanitizer build exposes older aggregate-initializer warnings in existing +nonportable test files. No sanitizer error was reported. + +Exact remaining gates: independent function-specific/platform accuracy matrix, +Windows/Linux/WASM and stock-published-pin wheel CI, publication/native pin +authorization, and a clean whole-tree docs gate (existing installation list-table +parsing errors, missing tutorial images, duplicate/generated documentation and +unrelated reference warnings remain). Development docs build succeeds and the +portable page has no reported diagnostics. Local draft integration is tested; +do not claim all beta/platform acceptance criteria complete. Final post-edit +Python/lint/docs/wheel results are recorded in the addendum below. + +### Exact implementation file inventory (preserved earlier work included) + +Python tracked changes: +`doc/reference/portable_dsl.rst`, `src/blosc2/b2objects.py`, +`src/blosc2/blosc2_ext.pyx`, `src/blosc2/core.py`, `src/blosc2/ctable.py`, +`src/blosc2/deserialization.py`, `src/blosc2/dict_store.py`, +`src/blosc2/dsl_artifact_bridge.h`, `src/blosc2/dsl_kernel.py`, +`src/blosc2/embed_store.py`, `src/blosc2/lazyexpr.py`, +`src/blosc2/msgpack_utils.py`, `src/blosc2/ndarray.py`, +`src/blosc2/portable_kernel.py`, `src/blosc2/schunk.py`, +`src/blosc2/tree_store.py`, `tests/ctable/test_ctable_computed_cols.py`, +`tests/ctable/test_ctable_dsl_columns.py`, `tests/ndarray/test_dsl_kernels.py`, +`tests/ndarray/test_lazyudf.py`, `tests/test_b2objects.py`, +`tests/test_deserialization.py`, `tests/test_dsl_portable.py`, +`tests/test_objectarray.py`. +New implementation files: +`src/blosc2/portable_lazy.py`, `tests/test_portable_descriptor.py`, +`tests/test_portable_descriptor_host.py`, `tests/test_portable_lazy.py`, +`tests/test_portable_policy.py`, `tests/test_portable_table_export.py`. +Durable task files: this progress document, the approved implementation plan and +scope inventory. Generated extension/build/wheel/log products are not source +implementation files; unrelated user untracked files remain untouched. + +Miniexpr tracked changes: +`CMakeLists.txt`, `src/miniexpr.h`, `src/dsl_portable.c`, +`src/dsl_artifact.c`, `src/dsl_compare.c`, +`src/dsl_compile.c`, `src/dsl_compile_internal.h`, +`src/dsl_compile_support.c`, `src/dsl_eval.c`, `src/dsl_eval_internal.h`, +`src/dsl_jit_runtime_host.c`, `src/dsl_jit_runtime_internal.h`, +`src/dsl_jit_runtime_nonhost.c`, `src/dsl_parser.c`, `src/dsl_parser.h`, +`src/functions.c`, `src/functions.h`, `src/miniexpr.c`, +`src/miniexpr_artifact.h`, `src/miniexpr_internal.h`, +`tests/test_dsl_artifact.c`, `tests/test_dsl_portable_validation.c`. +New miniexpr implementation files: +`doc/dsl-spec/1.0.md`, `doc/dsl-spec/artifact-1.0.md`, +`doc/dsl-spec/features-1.0.json`, `src/dsl_portable_expr.c`, +`src/dsl_portable_expr.h`, `src/dsl_portable_fp.h`, +`src/dsl_portable_types.c`, `src/dsl_portable_types.h`, +`src/dsl_semantic_profile.h`, `tests/test_dsl_portable_checked.c`, +`tests/test_dsl_portable_interp.c`, `tests/test_dsl_portable_types.c`. + +### Authored-save scope correction — latest verification (authoritative) + +This section supersedes all verification records below. No native implementation +changes were needed for authored-save conversion; the existing native 1.0 parser, +compiler and artifact validation remain authoritative. All checks used conda +`blosc2` and explicit development overrides where building wheels. + +- Full Python default suite: **11280 passed, 55 skipped**, 32.62s. Restored native + roundtrips for add-square, static loops/where, slices, named bindings, DictStore + operands, ObjectArray, captures/scalar constants, frames and nested arithmetic. + Complete logical-block reductions plus ND coordinates and sliced reads are + checked against independent expected values, retaining original partitions. +- Seven portable modules: **183 passed, 17 external skips = 200 cases**; tests + extend existing orthogonal cases rather than introducing another parameter grid. + Unsupported print/callback stores preserve old entries on Embed/Dict/Tree routes; + unsupported disk saves and loaded legacy recipe saves preserve destinations. +- Complete native suite: **343 passed**, 8.34s. Complete ASAN/UBSAN suite: + **343 passed**, 8.49s, `ASAN_OPTIONS=symbolize=0`. No sanitizer failures. +- Development-override wheel built and installed only to an isolated target; + isolated subprocess asserted package path and removed only its editable finder. + Portable modules plus b2objects/ObjectArray/ndarray DSL/CTable DSL/computed-column + modules: **458 passed, 17 external skips, 1 network deselected**, 4.22s. + Wheel `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `a4255253709584e7a1ccb69d585dc9ed6cc8a80d100b59a7bed0f6234f31e1c9`. + This is not stock-pin/platform certification. Editable import path and descriptor + availability were subsequently rechecked and preserved. +- Ruff check/format passed for the 10 Python files touched by this correction; + repository whitespace check passed. Sphinx build succeeds with **391 whole-tree + warnings**, including missing generated API stubs; the portable page itself has + no reported diagnostic. The clean whole-tree docs gate remains incomplete. +- Authoring route: `portable_from_lazyudf` in `src/blosc2/portable_lazy.py`, shared + through `b2objects.encode_b2object_payload` and LazyUDF save/frame. CTable APIs + accept DSLKernel plus `dtype`, explicit `row_domain='independent'` and optional + `cardinality='block_scalar'`; the approved plan and portable docs record this. + No legacy source fallback/migration or changed elementwise-to-scalar shape. +- Logs under the approved temporary root: `author-save-python.log`, + `author-save-portable.log`, `author-save-native.log`, `author-save-sanitized.log`, + `author-save-wheel-build.log`, `author-save-wheel-tests.log`, `author-save-docs.log`. + Wheel and isolated target directories: `author-save-wheel`, `author-save-wheel-smoke`. +- Local corrected scope is fulfilled. External function-specific accuracy, + Windows/Linux/WASM, authorized publication/pin and stock-wheel certification, + and clean whole-tree docs remain pending. No commits/push/publication occurred. + +### Public validator and finite-reference closure — earlier verification (superseded) + +These results supersede the earlier verification records below. All Python, +build and test commands used conda `blosc2`; native overrides were explicit. + +- Rebuilt the installed editable extension with the documented local miniexpr + and C-Blosc2 overrides. Rechecked `blosc2.__file__` resolves to this repository's + `src/blosc2/`, descriptor support is available, and 1.0 Unicode source validation + succeeds after both isolated wheel installations. Published pin unchanged. +- Full default Python suite: **11280 passed, 55 skipped**, 42.87s. +- Final complete native CTest: **343 passed**, 8.73s. Final complete ASAN/UBSAN + CTest: **343 passed**, 8.86s, `ASAN_OPTIONS=symbolize=0`. These include the + extended validator and independent analytic/exact reference fixtures, including + the final floating-power NaN exceptions. No sanitizer errors or new warnings + in the modified validator/interpreter tests were reported. +- Isolated development-override wheel: **183 passed, 17 external skips** across + the seven portable modules = **200 cases**. No new parametrized cases or local + 1.0 skips; validation assertions extend the existing authoring test. Wheel: + `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `39dc5fcd00867991a06dc14426829a305bccc3fa581224e5bb8dfc71b0e096bf`. + Isolated subprocess removed only its own editable finder, asserted wheel import + path, and ran pytest with `-n 0`. This is **not stock-pin certification**. +- Separately built/installed isolated artifact-disabled wheel: core frozen 0.1 + and draft 1.0 numeric/string/ND source validation passed, missing ND context + rejected, ordinary arithmetic passed, optional artifact import explicitly + rejected. Plain source validation does not require the optional JSON loader. +- Ruff check and format pass for all **27 changed/new Python files**; both + repositories' `git diff --check` pass. Finite feature JSON parses successfully. +- Sphinx HTML build succeeds with **142 whole-tree warnings**; no diagnostic + against `doc/reference/portable_dsl.rst` or the updated validator docstring. + Existing generated/reference/installation/image diagnostics are not a clean + documentation gate and remain recorded as outstanding. +- Logs under `/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode/`: + `closure-editable-build.log`, `closure-python.log`, `closure-native-build.log`, + `closure-native-ctest.log`, `closure-sanitized-build.log`, + `closure-sanitized-ctest.log`, `closure-wheel-build.log`, + `closure-wheel-tests.log`, `closure-disabled-build.log`, `closure-docs.log`. + Isolated wheel directories are `closure-wheel`, `closure-wheel-smoke`, + `closure-disabled-wheel`, and `closure-disabled-smoke`. +- Local requested implementation closure is complete. Remaining release gates: + independent function-specific/platform accuracy acceptance, Windows/Linux/WASM + execution, authorized native publication/pin and stock-pin wheel matrix, and + clean whole-tree docs. No beta-ready claim, publication, dependency change or + destructive repository operation was performed. + +### Earlier local verification addendum (superseded by closure results above) + +- `conda run -n blosc2 pytest -q`: **11280 passed, 55 skipped**, 38.69s, + after final row-shape/broadcast/width fixes. Subsequent test-helper/comment + cleanup reran ndarray DSL and CTable DSL tests: **138 passed**, 1.88s. +- Seven compact portable test modules: **183 passed, 17 skipped** = **200**; + the 17 skips are 16 external frozen-0.1 corpus cases and one external artifact + runner. No local draft 1.0 test skips. Frozen 0.1 self-contained artifact tests + continue to execute, with default export still 0.1. +- Complete native interpreter CTest: **343 passed**. Complete ASAN/UBSAN CTest: + **343 passed**, 43.90s, `ASAN_OPTIONS=symbolize=0`. Targeted artifact and + portable interpreter sanitizer pair: **2 passed**, 0.87s. No test failure or + sanitizer error; the existing nonportable aggregate initializer warnings noted + above do not indicate a new implementation warning. +- Ruff lint and format: all **27 changed/new Python files passed**. Cython is + validated by actual wheel compilation, not Ruff's Python parser. Both repos' + `git diff --check` passed. +- Final native-development override wheel: + `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `1d88d169e4a2af3df0747bdd5002deca019fab77aeac9b32aa3d40a1bb0f05a9`. + Built with explicit local miniexpr/C-Blosc2 source overrides and artifacts ON. + Installed into a separate temporary target; removed the editable import finder + only inside the smoke subprocess and asserted the loaded wheel package path. + The **entire 200-case portable suite** ran against that isolated wheel: + **183 passed, 17 external opt-in skipped**, 1.08s, including typed authoring, + table vector/string/constructor persistence and lazy descriptor tests. + This is **not** a published-pin/stock-wheel platform certification result. +- Separate artifact-disabled development wheel built and isolated smoke passed: + ordinary array arithmetic works, availability query is false, portable loader + explicitly raises `NotImplementedError`. Editable development install remained + untouched and was rechecked for the active descriptor ABI afterwards. +- Final incremental Sphinx HTML build succeeded with **117 whole-tree warnings**; + no diagnostic names `doc/reference/portable_dsl.rst` after correcting its new + heading underline. Earlier clean full build exposed existing installation + list-table parsing errors and unrelated missing-image/reference warnings. + A successful non-`-W` build is not a clean whole-tree documentation gate. + +Logs/builds/wheels remain under the approved temporary root +`/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode/`: +`portable-1-full-python-final.log`, `portable-1-comments-regression.log`, +`portable-1-native-ctest.log`, `portable-1-sanitized-all-ctest.log`, +`portable-1-sanitized-all-build.log`, `portable-1-docs-final.log`, +`portable-1-final-wheel/`, `portable-1-final-wheel-smoke/`, +`portable-1-disabled-wheel/`, `portable-1-disabled-smoke/`. +The supported local draft integration is implemented and tested. External +platform/independent-accuracy/stock-pin gates remain unexecuted, and no dependency +pin, publishing or repository state-changing git action was performed. + +### Independent correctness review addendum (2026-10-07) + +Scope: independently read repository instructions, approved scope/plan and current +handoff; inspected profile-aware expression typing/conversions, reduction masks, +scalar returns, constant decoding, artifact width/capability/cardinality and +descriptor checks, Python typed capture/export, lazy original-group reads and +persistence/policy propagation. This is focused local review, **not complete +assurance**, a whole-JIT audit or external platform certification. + +Concrete fixes: + +- `../miniexpr/src/dsl_eval.c`: portable `for` evaluation no longer zeroes the + iterator local in inactive lanes while another lane continues. Before the fix, + `for i in range(x): ...; return i` on `[2, 4]` returned `[0, 3]`, rather than + `[1, 3]`. Preserve full-DSL behavior outside the versioned portable profile. + Native regressions cover exhausted ranges, lane-specific break, and runtime + rejection of undefined iterators for empty ranges. +- `src/blosc2/portable_kernel.py`: determine integral capture magnitude with a + Python integer rather than `abs()` in the NumPy fixed-width dtype. Capturing + `np.int64(INT64_MIN)` as float64 formerly raised an overflow RuntimeWarning + (an exception under the suite's warning policy). Captured conversion and + immutable snapshot now have an authoring regression. +- Merged host boundary checks into existing portable cases: masked scalar return + through a reduction local with the first lane invalid, all-invalid sum identity, + tampered scalar cardinality/string output width rejection, and reverse/empty + elementwise partial reads of `x + sum(x)` preserving exactly the original group. + No test counts expanded, tests suppressed or failures relabeled. + +Findings from the initial review — resolved by the contract follow-up below: + +1. **Divergent block-scalar returns:** native import accepts + `s = sum(x); if x < 0: return s + 1; return s` as `block_scalar` (normalized + multiline DSL). Input `[-1, 2]` returns `1`: the first subset wrote `2`, then + the remaining subset overwrote the one scalar output with `1`. + `dsl_compile.c` recognizes the reduced local as uniform without accounting for + varying return control; `dsl_eval.c` writes each scalar return to the same slot. + The follow-up now rejects divergent scalar returns structurally; it neither + invents a selection rule nor reinterprets cardinality. +2. **Implicit legacy permission at the low-level MessagePack boundary:** + `msgpack_unpackb(payload)` in `msgpack_utils.py` deliberately defaults to FULL. + A handcrafted extension-43 legacy `lazyudf` record reconstructed a `LazyUDF` + without passing `deserialize='full'`. Safe-mode paths retain propagation and + rejection; this is not a demonstrated safe-mode bypass. The helper's documented + compatibility exception conflicted with the plan's explicit-opt-in wording. + The follow-up now defaults this boundary to safe and propagates explicit-full + permission through nested values, without a legacy/default-full fallback. + +Exact post-fix verification (conda `blosc2`, local native development override): + +- Rebuilt native interpreter, sanitizer build and Python editable extension using + the existing local miniexpr/C-Blosc2 FetchContent overrides; published pin unchanged. +- Full Python `pytest -q`: **11280 passed, 55 skipped**, **31.17s**, after final + boundary-test edits. Seven portable modules: **183 passed, 17 external skipped**, + **1.79s**, still exactly **200 cases**. +- Full native CTest: **343/343 passed**, **8.75s**. Full sanitizer CTest with + `ASAN_OPTIONS=symbolize=0`: **343/343 passed**, **8.91s**. No sanitizer error; + incremental review builds reported no warnings. A new exploratory test initially + assumed preassigned loop iterators were admissible; native compilation rejects + this existing syntax restriction. Replaced that assumption with the actual + supported empty-range/undefined-local boundary, not a change in language policy. +- Ruff lint and format checks pass for all four Python files changed by review; + both repos' `git diff --check` passes. No post-review wheel/docs rebuild claimed. +- Logs: approved temporary root, `portable-1-review-{native-build,native-ctest, + sanitized-build,sanitized-ctest,editable,full-python,portable-python}.log`. + +External beta gates remain independent function/platform accuracy certification, +Linux/Windows/WASM and stock-published-pin wheel CI, publication/pin authorization, +and clean whole-tree docs. The semantic/policy findings above were additional +local acceptance concerns and have now been resolved, as recorded below. +No commits, pushes, pin changes or publishing performed; unrelated files preserved. + +### Contract follow-up — both independent-review concerns resolved (2026-10-07) + +- `../miniexpr/src/dsl_compile.c` validates block-scalar return coherence after + static type/provenance joins converge. Reject returns under typed lane-varying + branches or loop bounds/conditions, including scalar returns in loops whose + lane-varying break/continue can split the return mask. Diagnostic: + `ambiguous block-scalar output: return under lane-varying control or loop flow`. + No first/last scalar selection and no output-cardinality reinterpretation. + Uniform reduced-scalar conditions/coherent scalar loops still compile; varying + loops may rejoin before a single top-level scalar return. Nested loop flow + rejoins locally rather than tainting unrelated outer returns. +- This intentionally conservative check also rejects matching scalar return + expressions inside a varying branch/loop: equality alone does not establish + coherent group control flow. The exact excluded patterns are documented in + `../miniexpr/doc/dsl-spec/1.0.md` and `doc/reference/portable_dsl.rst`. + Native and merged Python cases exercise divergent reduced-local returns, + varying range/while/elif, lane-varying break/continue, and accepted reduction- + controlled/coherent-loop and post-rejoin scalar returns. +- `src/blosc2/msgpack_utils.py`: public and private decoding boundaries default + safe. Explicit-full extension decoding passes its effective policy recursively + into frames, structured references, NumPy object values and sets. Audited all + source callsites: SChunk VLMeta/getall, EmbedStore metadata, ObjectArray and + BatchArray already pass their owning carrier's policy; proxy reads explicitly + pass safe. No internal call inherits the newly changed public default as a + substitute for caller policy. Locally constructed/full containers retain + their existing effective policy; safe-opened containers remain safe. +- Merged legacy MessagePack checks into the existing portable policy case: + default/safe handcrafted extension-43 legacy payloads, both direct and nested + in extension-46 object arrays, reject with a monkeypatched source-reconstruction + failure hook never called. Explicit-full decoding reconstructs those same + values intentionally. Existing local/full metadata and container tests pass + without adding default-full fallback or weakening any safe assertion. + +Final exact local verification: + +- `conda run -n blosc2 pytest -q`: **11280 passed, 55 skipped**, **34.60s**, after + final boundary checks. Seven portable modules: **183 passed, 17 external + skipped**, **3.10s**, exactly **200 cases**. Targeted descriptor/policy pair: + **10 passed**, **2.15s**. +- Rebuilt interpreter and sanitizer native builds. Complete CTest: + **343/343 passed**, **45.29s**; complete ASAN/UBSAN CTest with + `ASAN_OPTIONS=symbolize=0`: **343/343 passed**, **45.59s**. No sanitizer error or + new compiler warning; both incremental links reported the existing duplicate + `-lm`/`libminiexpr.a` warning. +- Rebuilt editable install with explicit local miniexpr/C-Blosc2 overrides; + rechecked installed source import and descriptor ABI after isolated wheel work. +- Built native-override wheel + `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `a0874703e46863b67040fbc096aa007c734e56f5f1b26da1fb903caa670cf649`. + Installed to a separate temporary target; removed the editable finder only in + the smoke subprocess and asserted the package path. Complete portable suite + against the isolated wheel: **183 passed, 17 external skipped**, **1.52s**, + using `-n 0` so worker startup cannot reintroduce editable imports. This is + local-development override evidence, **not stock-published-pin certification**. +- Ruff lint/format pass for all six Python implementation/test files changed by + independent review/follow-up; both repos' `git diff --check` passes. +- Sphinx command exits successfully; portable page has no diagnostic. Whole-tree + output includes **818 WARNING / 4 ERROR diagnostic lines**, including existing + installation list-table errors and unrelated reference/image issues. This is + not a clean docs gate, and no unrelated docs cleanup was attempted. +- Approved temporary-root logs use `portable-1-contract-` prefix: native/sanitized + build and CTest, editable, full/portable Python, wheel build/install/test and + docs. Wheel and target directories: `portable-1-contract-wheel/` and + `portable-1-contract-wheel-smoke/`. + +No remaining local semantic question from these two findings. External beta +gates remain independent function/platform accuracy certification, Linux/Windows/ +WASM and stock-published-pin wheel CI, authorized publication/native pin change, +and clean whole-tree docs. No backend expansion, commits, pushes or publication. + +## HISTORICAL RECORDS — superseded status/next-actions, retained verification history + +All inventory, pending-work and skip statements below describe earlier sessions. +They are not current handoff instructions. Use CURRENT STATUS above and its final +verification addendum for resume decisions. + +## Target and approved decisions + +Implement `portable-DSL-1.0-implemenation.md` through beta readiness. All Q1–Q3 +are approved: standard signed/unsigned integer widths and operand promotion; +explicit block-scalar cardinality/grid output; explicit `deserialize="full"` +legacy opt-in, without migration tooling. Interpreter correctness comes first. +No final contract freeze or publication is implied by completing implementation. + +## Starting state + +- Python-Blosc2: `23a2a91a`; no tracked modifications on entry. +- Miniexpr: `98b2bf9`; no tracked modifications on entry. +- Numerous unrelated untracked files exist in Python-Blosc2; preserve them. +- Native repository instructions: `../miniexpr/AGENTS.md` (K&R C style, minimal + changes, relevant tests, no new dependencies). +- Python commands must run through `conda run -n blosc2`. + +## Verification log + +- Baseline: `conda run -n blosc2 pytest tests/test_portable_artifact.py + tests/test_dsl_portable.py -q`: **159 passed, 41 skipped**. Skips include opt-in + external corpus/runner tests; this does not certify portable 1.0. +- Native build directory: + `/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode/miniexpr-portable-1-interpreter`. + Configured with `conda run -n blosc2 cmake -S /Users/faltet/blosc/miniexpr -B + -DMINIEXPR_ENABLE_TCC_JIT=OFF -DMINIEXPR_USE_SLEEF=OFF + -DMINIEXPR_USE_ACCELERATE=OFF -DMINIEXPR_BUILD_SHARED=OFF + -DMINIEXPR_BUILD_EXAMPLES=OFF -DMINIEXPR_BUILD_BENCH=OFF + -DMINIEXPR_BUILD_ARTIFACT=ON`; built with `--parallel 8`. +- First native suite with promotion helper: **341/341 passed**. Latest suite + adds checked-arithmetic tests: `conda run -n blosc2 ctest --test-dir + --parallel 8 --quiet --output-log /portable-1-native-ctest.log` + exited successfully. Current suite has 342 tests. +- Checked arithmetic compiled independently with `conda run -n blosc2 cc + -std=c99 -Wall -Wextra -Werror -fsanitize=address,undefined + -fno-sanitize-recover=all -I /Users/faltet/blosc/miniexpr/src + /Users/faltet/blosc/miniexpr/src/dsl_portable_types.c + /Users/faltet/blosc/miniexpr/tests/test_dsl_portable_checked.c + -o /portable-1-checked-sanitized`; executing it **passed**. + No compiler/sanitizer warnings. Full CMake build emits its existing duplicate + static-library/`-lm` linker warning; no new compiler warning was observed. +- Draft matrix audit: all **87** distinct registered `functions.c` builtin names + are covered, plus four DSL intrinsics (`int`, `float`, `bool`, `range`); no + duplicate matrix names. This is coverage of the inventory, not certification. +- `git diff --check` passes in both repositories. + +## Milestones + +1. In progress: implemented draft schema and finite builtin inventory; concrete + platform/domain/tolerance certification remains (not a schema design blocker). +2. Integrated: profile-aware typed interpreter and draft artifact 1.0 acceptance + for validated implemented capabilities; standalone 0.1 validator stays frozen. +3. Integrated: numeric operations/casts/comparisons/lazy operands and static local + joins. Remaining work is the finite conformance/platform matrix. +4. Integrated: ordered masked reductions, padded lanes, scalar cardinality, + empty standalone blocks and evaluation-local caches/definition masks; expand + returned-lane/control-flow/concurrency certification fixtures. +5. Integrated: extended artifact descriptor ABI, typed signatures/constants and + immutable width/cardinality/capability/context validation. +6. Integrated: fixed strings, dynamic integral indices/bounded replacement, and + logical ND coordinates; platform/endian/Unicode conformance remains. +7. In progress: conditional Python bridge and standalone descriptor routing done; + logical lazy scheduling, partial reads and compiled Python 1.0 tests remain. +8. Pending: all persistence routes and legacy policy enforcement. +9. Integrated: interpreter-only optional-JIT fallback and explicit required-JIT + unsupported diagnostic; frozen/full profiles preserve their original behavior. +10. Pending: complete conformance/platform/wheel/docs verification. + +## Findings to retain + +- `functions.c` registers `log` differently depending on `ME_NAT_LOG`; the + portable profile must bind natural log explicitly, independent of build flags. +- Existing artifact eval assumes elementwise `nitems` output and skips empty + calls. Extended descriptors are required for block-scalar and empty identities. +- Existing `b2objects.py` internally hard-codes `deserialize="full"` on operand + opens. Effective user policy must propagate instead. +- Full-DSL compilation uses shared expression helpers; approved operand typing + must be profile-aware, not an unversioned global change. + +## Next action + +Continue with static local joins, remaining numeric functions and mask conformance, +then extended artifact descriptors/cardinality/context. +Do not enable 1.0 acceptance until the typed-tree/evaluator actually enforces the +contract. Current public runtime APIs still accept only 0.1; staged semantics are +isolated from frozen/full-DSL execution. No semantic blocker identified yet. + +## Continuation increment: typed interpreter integration + +- Added internal `dsl_compile_program_profile()` and `private_compile_profile_ex()`; + the original entry points retain full-DSL behavior. Profile metadata is per + program/parser state, not global. The 1.0 raw-tree path bypasses full-DSL numeric + promotion, optimization, bytecode and output-context node retyping. +- Parser leaves preserve exact decimal integer magnitudes (through uint64), weak + literal identity and direct float32 literal rounding. Unary literal signs are + resolved without executing arithmetic. Typed pass inserts checked conversion + nodes, including final output conversion only after operand-driven computation. +- Added `dsl_portable_expr.[ch]` operating on the existing `me_expr` tree and + checked helpers. Core arithmetic, shifts/bitwise, powers, exact mixed integer + comparisons, int/float/bool casts and lazy truth/where now execute natively. + Unsupported operations still reject in this **internal staged** path. +- DSL assignment/return and condition masks route through the checked evaluator; + skipped lanes do not execute invalid arithmetic. New locals infer independently + of output dtype. Existing locals still use their first inferred type; approved + static join inference remains to implement. +- Optional JIT requests are forced to interpreter before IR construction in 1.0. + Print is rejected. No public 1.0 dispatch or persistence changed yet. +- Reconfigured/rebuilt the native build above; `tests/test_dsl_portable_interp` + passed. `conda run -n blosc2 ctest --test-dir --parallel 8 --quiet + --output-log /portable-1-native-ctest.log` passed (343 tests). +- Additional changed native files: `functions.[ch]`, `miniexpr.c`, + `dsl_compile.c`, `dsl_compile_internal.h`, `dsl_jit_runtime_internal.h`, + `dsl_eval.c`, `dsl_semantic_profile.h`, `dsl_portable_expr.[ch]`, + `tests/test_dsl_portable_interp.c`, and the CMake source list. +- This initial increment's tests were subsequently expanded and explicitly made + assertion-enabled even in Release builds; use the latest verification below. + +## Continuation increment: ordered masked reductions and numeric coverage + +- Profile-aware DSL parsing retains `//=` instead of lowering through floating + `floor(a / b)`. Expression tokenization and comparison lowering recognize `//`; + full-DSL tokenization/compound-assignment behavior stays unchanged. Checked + integer floor division/remainder, float floor division/divisor-sign remainder, + and distinct truncation-sign builtin `fmod` now execute through typed nodes. +- Added explicit builtin identity dispatch for the staged unary libm family, + aliases, selected binary libm functions, abs/square/sign and real-only + conj/real/imag. Calls use float32 libm entry points for float32 computation and + float64 otherwise; no external callback is executed. Natural-log binding is + independent of `ME_NAT_LOG`. `rint` ties-to-even does not use host rounding mode; + `round` keeps native half-away ties. Scalar fmin/fmax zero ties are explicit. +- sum/prod/mean/min/max/any/all typed nodes now reduce current participating lanes + in logical order, with checked integer accumulation and float32 operation + boundaries. Empty identities/errors, numeric NaN propagation and min/max zero + ties are implemented. A per-expression evaluation-local reduction cache + broadcasts without quadratic repeated reduction work for normal expressions; + it is never stored on a compiled/shared handle. +- Expanded native integration coverage: exact int64/uint64 literals and + comparisons, contextual literal errors, arithmetic/shift/power extrema, + operand-vs-output widths, narrowing, NaN truth/comparisons, libm typing, + conditional invalid arithmetic, nested masked reductions, loop early exit, + returned lanes, continue, empty identities and ordered cancellation/overflow. +- Added profile guards to `dsl_jit_runtime_host.c`, + `dsl_jit_runtime_nonhost.c`, and both execution paths in `dsl_eval.c` so no + uncertified cached/backend function can execute for a 1.0 program. +- Staged compilation rejects external callbacks, print, non-strict FP, ND symbols + without the unfinished explicit descriptor, unsupported calls/strings, nested + reductions and reduction-valued short-circuit/where alternatives needing + additional expression masks. These are **temporary implementation diagnostics**, + not approved scope removals. Local dtype-changing reassignment rejects clearly + rather than silently narrowing to the first assignment. Static joins remain + required. Inconsistent return cardinality now rejects. +- Additional changed native files: `dsl_parser.[ch]`, `dsl_compare.c`, + `dsl_jit_runtime_host.c`, `dsl_jit_runtime_nonhost.c`, and updated draft `1.0.md`. + +### Latest reproducible verification + +Let `T=/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode`. +All commands ran through the required `blosc2` environment. + +- `conda run -n blosc2 cmake --build $T/miniexpr-portable-1-interpreter --parallel 8` + passed; log `$T/portable-1-build.log`. Normal build has only the previously + observed duplicate-library linker warning. +- `conda run -n blosc2 ctest --test-dir $T/miniexpr-portable-1-interpreter + --parallel 8 --quiet --output-log $T/portable-1-native-ctest.log`: **343 passed**. +- Integrated sanitizer build configured with: + `conda run -n blosc2 cmake -S /Users/faltet/blosc/miniexpr + -B $T/miniexpr-portable-1-sanitized -DMINIEXPR_ENABLE_TCC_JIT=OFF + -DMINIEXPR_USE_SLEEF=OFF -DMINIEXPR_USE_ACCELERATE=OFF + -DMINIEXPR_BUILD_SHARED=OFF -DMINIEXPR_BUILD_EXAMPLES=OFF + -DMINIEXPR_BUILD_BENCH=OFF -DMINIEXPR_BUILD_ARTIFACT=ON + -DCMAKE_BUILD_TYPE=Debug + '-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-sanitize-recover=all -Wall -Wextra'`. + `conda run -n blosc2 cmake --build $T/miniexpr-portable-1-sanitized + --target test_dsl_portable_interp --parallel 8` and + `conda run -n blosc2 $T/miniexpr-portable-1-sanitized/tests/test_dsl_portable_interp` + **passed**, with no sanitizer findings. The warning-enabled shared-source build + emits numerous existing unused-function/initializer warnings; no warnings came + from the new expression module. Log `$T/portable-1-sanitized-build.log`. +- Independently: `conda run -n blosc2 cc -std=c99 -Wall -Wextra -Werror -I src + -c src/dsl_portable_expr.c -o $T/portable-1-expr-warning-check.o` **passed**. +- Python regression command from the baseline rerun: **159 passed, 41 skipped**. + It uses the installed/pinned native dependency, not sibling-checkout inference; + it does not verify staged 1.0 or newly edited native code. +- Latest `git diff --check` passes in both repositories. + +### Handoff and true remaining gates + +No unresolved approved-semantic conflict or external blocker was encountered. +The implementation is coherent/tested but **not beta-ready**. No public 1.0 +validator/artifact acceptance, Python native bridge/persistence transition or pin +update was made; no release/publication/platform-CI result is claimed. + +Next concrete work: + +1. Add static numeric local joins (do not reintroduce output-context typing), + finish remaining matrix operations (`fma`, `ldexp`, `logaddexp`, e/pi and + combinatorics), per-function domains/tolerances and bound/range conversion + conformance. Review the FP environment/nearest-rounding requirement explicitly. +2. Extend expression masks for reduction-valued lazy operands if admitted; + complete Mandelbrot/partition/padding/concurrent-handle fixtures and cardinality + joins. Existing program eval still broadcasts scalar outputs into nitems slots + and does not execute an empty standalone block; do not certify that old ABI. +3. Implement extended native artifact schema/API with capacities, valid lanes, + logical context and block-scalar outputs. Only then admit strings/ND and public + 1.0 validation/loading, retaining frozen 0.1 dispatch. +4. Continue Python bridge, immutable logical block scheduling, all persistence + routes and effective explicit legacy opt-in, then the remaining beta gates. + Do not update the published native pin to unpublished local code. + +## Further continuation: static joins and strict FP environment + +- Implemented bounded fixed-point numeric local inference using discovery trees + that are discarded, followed by rebuilding against stable slot dtypes. Numeric + return paths also join when output dtype is inferred. Storage conversion is + inserted after RHS operand-driven computation, not before it. Frozen/full DSL + continues its original single-pass compilation. For-variable collision checks + are preserved while permitting discovery-pass rebuilds. +- Added `dsl_portable_fp.h`: staged compilation and expression evaluation save + the caller thread's floating environment, establish nontrapping IEEE nearest/ + ties-even, then restore rounding and exception flags on success/failure. No + shared handle or global semantic switch stores this state. +- Added local reassignment/loop/return-join tests and compilation/evaluation under + FE_UPWARD with pre-existing FE_INVALID, checking nearest arithmetic/literals and + exact caller environment restoration. Target `test_dsl_portable_interp` rebuilt + and **passed** in the ordinary native build. Integrated sanitizers/full suite + will be rerun after the next dependent increment. +- Supersedes the earlier temporary "local type joins not implemented" diagnostic + and nearest-rounding TODO. Remaining operation, masks, descriptor and host gates + are unchanged; public 1.0 acceptance remains closed. + +## Further continuation: remaining numeric operations and lazy reduction masks + +- Added typed fma/fmaf (explicit fusion), integral-exponent ldexp with checked int + bounds, stable logaddexp including equal infinities, exact e/pi constants, and + checked integral fac/ncr/npr. Combinatorics retains the promoted integral dtype; + negative operands or k>n error. Binomial factors cancel before multiplication, + so e.g. uint64 ncr(67,33) succeeds without spurious intermediate overflow. +- Found native pure FUNCTION0 token value 40 collides with logical-not token 40. + Portable tokenization preserves zero-argument builtin identity without that + token flag; frozen/full behavior is not changed. +- Reduction-valued where alternatives and short-circuit RHS now have explicitly + derived participating masks, with evaluation-local child contexts/caches and + ownership cleanup. Discarded arithmetic/reductions are not executed. Generated + synthetic short-circuit assignments are admitted structurally while retaining + the prohibition on user reduction assignments/returns in control-flow bodies. +- Compact integration cases cover checked factorial, exact large binomial, + explicit fused cancellation, ldexp, infinite logaddexp, constants, masked + where/and reductions and nested alternative masks. Ordinary native target + `test_dsl_portable_interp` rebuilt and **passed**; sanitizer/full rerun follows. +- Supersedes the temporary reduction-valued lazy-operand diagnostic. All numeric + builtin names now have staged implementations (complete domain/accuracy corpus + and finite per-operation matrix details are still certification work). + +## Further continuation: descriptors, empty groups, definition state and fixtures + +- Added internal `dsl_eval_program_portable()` with a byte-capacity descriptor, + valid-lane mask, single-item block-scalar output and explicit empty scalar-block + execution. Reduction provenance propagates through locals; lane-dependent + condition/range/loop-flow assignments demote uniformity. Empty reductions use + the real zero-lane group, with a separate dependency slot for scalar locals. + Elementwise zero-item calls still skip execution. Legacy eval ABI is unchanged. +- Added per-call/per-lane local initialization masks to staged numeric evaluation; + participating uninitialized reads error, lazy discarded reads do not. Loop + variable writes define only active lanes. Range arguments now use checked int64 + conversions instead of legacy unchecked float/uint64 casts. +- Added public extended **descriptor ABI** (not language/schema 1.0 loading): + `me_artifact_eval_ex`, buffer widths/capacities, logical-context/mask fields, + cardinality and itemsize queries. Frozen 0.1 rejects unavailable context/masks. + Product/address extent/alignment/overlap checks precede execution; old ABI is + preserved. Tests cover undersized/misaligned/overlapping buffers, bad descriptor + versions, overflow and empty calls with no data/output buffers. +- FP environment setup now also verifies gradual float32/float64 underflow, + rejecting flush-to-zero/denormals-are-zero environments without arch-specific + or global mutable switches. Updated native language/artifact drafts. +- Compact fixtures added for scalar output sentinels, empty sum/mean/min, local + scalar provenance, definition masks, invalid range bounds, lane-dependent + cardinality, Mandelbrot-style break/partition/padding, and concurrent shared + handle calls with distinct caller rounding/exception flags. +- Descriptor/artifact and initial empty-group increment passed all **343** native + tests; targeted interpreter/artifact ASAN/UBSAN tests both passed. Subsequent + definition/range/FP/fixture/cardinality edits are being verified in the next + full/sanitizer rerun. No public 1.0 acceptance or native pin update occurred. + +- Latest integrated rerun after definition/range/gradual-underflow fixtures and + lane-dependent cardinality demotion: full **343/343 native tests passed**; + targeted artifact/interpreter ASAN/UBSAN **2/2 passed**; the typed expression + module still compiles with `-Wall -Wextra -Werror`; Python baseline rerun is + **159 passed, 41 skipped**; both repository `git diff --check` calls passed. + Ordinary native build retains only the observed duplicate-library linker + warning. Follow-up concurrent error-path FP restoration assertions and the + sinpi/cospi increment below are also covered by the final integrated rerun. + +## Further continuation: pi-scaled math accuracy + +- Replaced staged sinpi/cospi's large `pi*x` intermediate with exact modulo-two + reduction and reflection in pi units before libm evaluation. Integer and + half-integer anchors are exact, including large float32/float64 inputs; near + half-integer cospi avoids cancellation. Documented signed-zero anchors and + non-finite NaN behavior; frozen/full DSL is untouched. +- Added integer/half-integer/large/negative-zero/near-half/infinite input fixtures + for both float precisions. Concurrent shared-handle fixtures now also assert + FP environment restoration after checked-arithmetic failure. +- Final verification: **343/343 native tests passed**, targeted ASAN/UBSAN + interpreter/artifact **2/2 passed**, typed expression source warning check + **passed** with `-Wall -Wextra -Werror`. Python regression remains the latest + **159 passed / 41 skipped** run above. No genuine blocker occurred; beta gates + remain implementation work, not a scope reduction or public 1.0 certification. + +### Precise remaining work after these increments + +1. Finish static cardinality/placement conformance (including returned lanes and + reduction-valued nested flow), per-operation domain/tolerance/type matrix, and + platform-specific strict-FP certification. Extended internal evaluator has not + been certified as a complete artifact execution contract. +2. Connect descriptor-based numeric 1.0 artifact compilation/evaluation only after + validation, typed constants and immutable signature/cardinality are enforced; + finish actual schema 1.0 decoding and required-context/capability queries. +3. Add strings and ND using existing machinery with profile-aware semantics; + verify byte widths, capacities, coordinates and empty logical blocks. +4. Then complete Python bridge/logical scheduling/persistence/legacy policy and + platform/wheel/documentation gates. Keep published pin and public version gates. + +## Continuation: native schema/signatures, strings, ND, and staged Python bridge + +- `me_artifact_load` now dispatches schema/language 1.0 to profile-aware typed + interpreter compilation. Immutable signatures enforce all eleven numeric types, + exact width-checked signed/unsigned constants, IEEE bits, fixed-string widths, + Unicode32BE/byte hex constants, result cardinality and context rank. Unknown, + missing and unimplemented capabilities/operations/signatures explicitly reject. + Source tree scanning requires used numeric/control/reduction/string features. + Draft acceptance is not full certification; 0.1 validator/ABI remain frozen. +- Added internal uniform-input signature marker for strong captured constants; + constants are broadcast in call-local buffers without mutating shared handles. + Descriptor evaluation bypasses legacy tile execution/JIT; legacy eval rejects + 1.0 handles. Added native schema/capability/rank queries. +- Reused existing fixed-string validators, width inference, Unicode 15.0.0 case + tables and kernels through stack-local materialized operand adapters, not a new + parser/engine. Typed nodes handle string values, nested operations, predicates, + concat, lazy where and string-width fixed-point joins. Non-string numeric ops + reject string operands; explicit bool/string truth is defined by first code unit. + Active Unicode input scalars validate; masked padded inputs are not read. +- ND coordinates come only from explicit descriptor shape/origin/extent: checked + row-major logical `_flat_idx`, `_iN`, logical `_nN`, rank. Extent product equals + nitems; out-of-domain padded coordinates require invalid masks. Context buffers + and masks cannot overlap output. There is no implicit physical/global synthesis. +- End-to-end native artifact fixtures cover masked scalar/empty results, exact + uint64 max constants, missing capabilities/cardinality/rank, thirteen byte-string + operations (nested/where), Unicode expansion/truncation/constant endian encoding, + invalid Unicode, and padded 2D coordinates. Target ordinary native tests pass. +- ASAN found an adapter header copy over-read of zero-arity expression allocations; + fixed by copying only `offsetof(me_expr, parameters)` into owned stack headers. + Sanitized artifact executable now passes with `ASAN_OPTIONS=symbolize=0` (the + local symbolizer stalled when reporting the now-fixed finding). Interpreter + sanitizer target also passed. Final full rerun follows below. +- Started conditional Python native bridge: descriptor availability, authoritative + descriptor_info, and explicit evaluate_block (scalar shape(), fixed widths, + masks and logical ND context). Cython generation passed; new optional installed- + native tests never search a sibling checkout. No native pin/build replacement. + +### Exact next task at session limit + +1. Add returned-lane/reduction-local/string concurrent fixtures and finish finite + operation domain/type matrix certification (special domains, float32 result + types, literal/strong operand combinations, endian and width bounds). Dynamic + integral substr/split indices, replace bounds for typed constants/dynamic + strings, and explicit required-JIT rejection were completed subsequently. +2. Complete logical block-grid scheduling and partial reads in the lazy host. + `PortableKernel.from_json`, signature/cardinality/rank metadata, standalone + evaluate and explicit evaluate_block now route to native descriptors when + the installed ABI supports them, without Python reconstruction. Authoring 1.0, + lazy scheduling and persistence remain unchanged. Preserve pin independence; + actual compiled Python/native 1.0 execution tests currently require an opt-in + runtime and skip normally. Low-level Cython generation and bridge header compile + pass; host routing is tested with an explicit test double, not a language engine. +3. Complete all safe persistence routes and explicit legacy policy propagation; + then platform/wheel/docs gates. No genuine blocker or scope conflict occurred. + +- Subsequent native signature increment: integral source/captured/per-lane string + indices materialize into existing kernels with safe bounded clamping; + substring dynamic lengths retain subject width. Replace derives a checked + literal-aware/worst-case bound from capacities, rejecting widths over one MiB + and empty participating needles. Added negative index, UINT64_MAX length, + typed byte constants/replacement and runtime-empty-needle fixtures. Ordinary + and ASAN/UBSAN artifact executables passed. Added explicit `jit-required` + unsupported diagnostic; final integrated rerun follows. + +## Latest integrated verification and resume boundary + +### Current continuation: logical partitions and persistence (local native build) + +- Rebuilt the editable extension in conda `blosc2` using supported CMake + `FETCHCONTENT_SOURCE_DIR_MINIEXPR=/Users/faltet/blosc/miniexpr` and + `FETCHCONTENT_SOURCE_DIR_BLOSC2=.../build-portable-python/_deps/blosc2-src` + development overrides, `pip install -e . --no-build-isolation`. Published + FetchContent pin is unchanged. All three installed-native descriptor tests + now execute and pass; they do not skip in local verification. +- Added `src/blosc2/portable_lazy.py` native-only `PortableLazyArray`, constructed + via `PortableKernel.lazy`. Logical domain/partition tuples are independent of + operand storage grids. Scalar returns allocate the C-order partition grid; + elementwise returns allocate the original domain. Edge groups use actual valid + extents (no padded storage lanes synthesized). ND calls preserve original + logical shape/origin. Native partition lane limits validate at construction. +- Basic slices/integers/ellipsis/negative strides select intersecting original + groups, evaluate each whole group, and only then gather selected output lanes. + Partial scalar reads never repartition a reduction. Fancy indexing conservatively + evaluates the full original result before indexing. `compute`/`rechunk` preserve + partitions while choosing output storage chunks/blocks. Full compute streams + output storage chunks using selected original groups, avoiding full-result + NumPy allocation. Groups crossing output chunk boundaries may be evaluated + repeatedly but are never repartitioned. Partial/fancy reads return NumPy values. + Broadcast operands materialize against each group's extent while preserving + output-domain ND coordinates, with a partial-read regression fixture. +- Portable disk carriers, cframes, MessagePack structured objects, EmbedStore, + DictStore, TreeStore and nested arithmetic LazyExpr payloads roundtrip with + native artifact JSON and logical partitions, no originating function/source + reconstruction. Native artifact load precedes operand binding. Normal tests + never infer a sibling directory; local overrides are development-only. +- Removed new full/Python LazyUDF persistence and the old source fallback. + Shared recursive preflight runs before store overwrite/delete, and LazyUDF + save/frame rejects before touching a destination. Raw legacy UDF carriers + reject on NDArray save/frame too. Existing source-backed files still load only + under explicit effective `deserialize='full'`; policy propagates recursively + through structured operand refs and old LazyArray refs. No migration added. +- Safe arithmetic LazyExpr envelopes now have an AST whitelist (no calls, + attributes, subscriptions or ambient names) before host construction, with + safe policy propagated to every nested operand. Legacy nested DSL cannot hide + behind an arithmetic or portable carrier; explicit full remains required. +- CTable computed/generated legacy DSL metadata is policy-detected before + reconstruction; persistent add/save/frame/copy/physical-pack routes reject + full-DSL source descriptors before output writes. In-memory full DSL computation + remains supported. **Native portable CTable computed/generated transformer + descriptors are still unfinished**; do not claim table integration complete. +- Current new tests cover original-group reads, physical grid mismatch, partial + slices/rechunking, block-scalar grid shapes, ND/Unicode/mixed disk roundtrips, + all five container routes, native-only guard, preserved overwrite destinations, + effective nested legacy policy and CTable policy/preflight. Existing legacy + roundtrip tests now assert the new save rejection, while frozen legacy load + fixtures remain tested with explicit full. Removed 22 redundant external-corpus + Python parameter cases; exhaustive conversion coverage remains native. + +Changed Python inventory added this continuation: `src/blosc2/portable_lazy.py`, +`portable_kernel.py`, `b2objects.py`, `lazyexpr.py`, `schunk.py`, `core.py`, +`ndarray.py`, `msgpack_utils.py`, `embed_store.py`, `dict_store.py`, `tree_store.py`, +`ctable.py`; tests `test_portable_lazy.py`, `test_portable_policy.py`, +`test_b2objects.py`, `test_deserialization.py`, `test_dsl_portable.py`, and +`ctable/test_ctable_computed_cols.py`. Native edits from previous increments +remain preserved; no publishing, pin change or git state operations occurred. +User documentation updated in `doc/reference/portable_dsl.rst` for draft logical +partitions, partial reads, persistence routes and explicit legacy policy. + +Exact next implementation: portable CTable computed/generated descriptors and +row-domain/append/refresh semantics. Full DSL table sources now reject new +persistence but native portable table transformers are not connected yet. Then +finish authoring 1.0 records and the finite native conformance/platform gates. + +Latest verification for this continuation: + +- Combined relevant Python portable/persistence/CTable suite: **315 passed, + 19 skipped**. All actual 1.0 descriptor/lazy/ND/string runtime tests execute, + none skip. The 19 skips are 18 opt-in external 0.1 corpus fixtures and one + opt-in standalone native artifact runner; stock tests still do not discover + sibling sources. Portable-only coverage remains compact after replacing + redundant conversion/corpus parameter cases with orthogonal host boundaries: + **181 passed + 19 skipped = 200 portable Python cases** in the focused run. +- Full native CTest rerun passed (**343 tests**); targeted ASAN/UBSAN artifact and + interpreter rerun **2/2 passed** with symbolization disabled as documented. +- Ruff lint/format for all 18 affected Python source/test files passed; both repo + `git diff --check` calls passed. Local editable build completed successfully; + the three previously skipped installed-native descriptor fixtures passed. +- CTable computed/generated tests now **96 passed**, keeping full DSL local + computation while asserting new source persistence rejection. Frozen legacy + file load remains tested under explicit full. No external platform CI or wheel + matrix executed; do not claim beta readiness. No semantic blocker encountered. + +- Final native build and full CTest run: **343/343 passed**, including all new + schema/signature/string/ND/index/replacement fixtures. Build log retains only + the existing duplicate-library linker warning. +- Final targeted ASAN/UBSAN interpreter/artifact CTest run: **2/2 passed** with + `ASAN_OPTIONS=symbolize=0` to avoid the local symbolizer hang. The real adapter + over-read was fixed before this passing run; it is not being suppressed. +- `dsl_portable_expr.c` and `dsl_artifact.c` independently compiled with + `-Wall -Wextra -Werror`. Cython generation and bridge header syntax checks for + both descriptor-enabled and unavailable branches passed. This is not yet an + end-to-end compiled Python 1.0 runtime certification. +- Python relevant tests: **160 passed, 44 skipped**; three additional skips are + intentional installed-runtime 1.0 descriptor fixtures. The extra passing host + fixture checks native-authoritative metadata, byte-order/stride normalization + and explicit block routing without a source interpreter/test engine. +- Ruff lint/format and both repository diff whitespace checks passed. Source + byte-string literals now contextualize safely in where/output families, with + ASCII checks and bounded UTF32-to-byte materialization; native fixture passes. +- First resumed implementation task: connect `PortableKernel` standalone block + routing to lazy **logical partition scheduling** (immutable C-order block grid, + block-scalar grid result shape, full-group evaluation before partial selection, + explicit ND origin/extent and padding masks), then persistence/legacy policy. + Keep old installed runtime tests independent. In parallel with those dependency + milestones, finish the finite conformance/platform checklist above, not another + numerical approximation polishing loop. No genuine blocker encountered. + +## Files changed in this implementation session (current inventory) + +Python-Blosc2: + +- `plans/portable-DSL-1.0-progress.md`: durable handoff and verification. +- `src/blosc2/dsl_artifact_bridge.h`: version-guarded descriptor ABI/fallbacks. +- `src/blosc2/blosc2_ext.pyx`: low-level signature and block-evaluation bridge. +- `src/blosc2/portable_kernel.py`: authoritative descriptor metadata/standalone + block routing with explicit ND context, no Python source reconstruction. +- `tests/test_portable_descriptor.py`: optional installed-native 1.0 fixtures. +- `tests/test_portable_descriptor_host.py`: pin-independent host routing fixture. +- Previously authored scope/implementation plans and unrelated untracked work + remain preserved. No native pin, public authoring or persistence changes yet. + +Miniexpr: + +- `CMakeLists.txt`: add typed helper/expression sources and native integration tests. +- `src/dsl_semantic_profile.h`: per-program full/portable profile identities. +- `src/dsl_portable_expr.h`, `.c`, `dsl_portable_fp.h`: typed tree validation, + checked interpreter, libm/reductions/masks/string values and local FP policy. +- `src/dsl_compile.c`, `dsl_compile_internal.h`, `dsl_compile_support.c`: + profile compilation, fixed-point local joins/widths/cardinality, ownership. +- `src/dsl_eval.c`, `dsl_eval_internal.h`: masked statement integration, + initialization tracking, scalar/empty/descriptor/ND evaluation. +- `src/dsl_parser.c`, `dsl_parser.h`: profile token/parser integration. +- `src/functions.c`, `functions.h`, `src/miniexpr.c`, `miniexpr_internal.h`: + raw typed expression profile, operator identities, string-kernel adapters. +- `src/dsl_compare.c`: profile-aware integral comparison integration. +- `src/dsl_jit_runtime_internal.h`, `dsl_jit_runtime_host.c`, + `dsl_jit_runtime_nonhost.c`: staged interpreter/JIT guards and program metadata. +- `src/dsl_artifact.c`, `miniexpr_artifact.h`: draft schema/signatures/constants, + capabilities/context/cardinality queries and checked descriptor API. +- `tests/test_dsl_artifact.c`, `test_dsl_portable_interp.c`: end-to-end fixtures. +- `src/dsl_portable_types.h`, `src/dsl_portable_types.c`: internal operand + promotion and division dtypes; checked signed/unsigned integer arithmetic, + floor division/modulo, shifts, nonnegative powers, fixed-width bitwise + operations, float-to-integer and exact integer conversions/comparisons. +- `tests/test_dsl_portable_types.c`: explicit 121-pair expected promotion matrix. +- `tests/test_dsl_portable_checked.c`: exhaustive small-domain and extrema tests. +- `doc/dsl-spec/1.0.md`, `artifact-1.0.md`, `features-1.0.json`: native drafts; + not a claim of implemented 1.0 or a frozen specification. + +Helpers define left shift as checked multiplication, signed right shift as floor +division by a power of two, and nonnegative integer powers (`0**0=1`). Fractional +weak floating literals next to integers/Booleans remain floating rather than +being truncated. These are implementation-led details documented in the draft. diff --git a/plans/portable-DSL-1.0-scope-inventory.md b/plans/portable-DSL-1.0-scope-inventory.md new file mode 100644 index 000000000..4563f230f --- /dev/null +++ b/plans/portable-DSL-1.0-scope-inventory.md @@ -0,0 +1,245 @@ +# Portable DSL 1.0: agreed scope and specification work + +Status: feature-level scope agreed on 2026-10-07. Version 1.0 is the intended +first public compatibility contract; its detailed specification remains draft +until the acceptance gates below pass. Draft 1.0 is the sole public portable +format/default; the unreleased 0.1 path is retired with unsupported-version rejection. + +## Goal + +Persist DSL kernels only when their normalized source, typed bindings, and +execution requirements have a language-independent contract. Validate on save +and independently on load. Preserve convenient authoring syntax when it can be +lowered without retaining Python execution or live Python state. + +Interpreter-first: missing JIT support is not grounds for excluding a feature. +Select fallback before execution. A valid artifact can require capabilities +which a particular host lacks; that host must reject it explicitly. + +## Agreed inclusions + +- Real/integer/Boolean arithmetic, mixed numeric inputs, casts, predicates, + and an explicitly enumerated math-function set, including elementwise `where`. +- Existing branches, loops, local assignments, and loop control. +- ND index/shape symbols with explicit host-supplied logical context. +- Fixed-width Unicode strings/bytes and statically bounded string returns. +- Supported authoring rewrites and immutable typed scalar captures; static + column references become explicit named bindings. +- Limited block-local reductions: top-level reductions and supported + control-flow conditions, including Mandelbrot-style `all(...)` early exit. +- Portable kernel persistence with validation after normalization on save and + independent native validation on load. Unsupported kernels must fail saving; + no silent Python-specific persistence fallback for this path. +- Interpreter-first semantics; JIT is optional and may fall back before execution. + +## Inventory: prioritize semantic difficulty + +| Feature | Current implementation evidence | Portability work / difficulty | Agreed feature-level disposition | +| --- | --- | --- | --- | +| Block reductions inside kernels | Native DSL admits top-level reductions and reductions in conditions; rejects reductions inside control-flow bodies (`src/dsl_compile.c`). | **High.** Explicit block grouping, masking, scalar outputs/broadcast, accumulation order, empty blocks and padding. | Include limited reductions over the current evaluation block, including Mandelbrot `all` conditions. Exclude unsupported reductions inside control-flow bodies; no implicit full-array reduction. | +| External functions and closures | Native `me_variable` supports registered C functions/closures. Python authoring allows named calls syntactically; native compilation determines availability. | **Very high.** Function implementation identity/version, state, ABI, registration, and reproducibility cannot be represented by process-local pointers. | Exclude externally registered functions, callbacks, and opaque captured objects. Allow supported builtin functions and typed constant snapshots. | +| Complex arithmetic | Native complex types exist, but Windows rejects them because of complex ABI incompatibilities (`doc/data-types.md`, `doc/windows-limitations.md`). | **High.** Portable representation, branch cuts, non-finite behavior, promotion, and a functioning interpreter across target platforms. | Exclude complex inputs, intermediates, outputs, and constants for 1.0. This is a real reduction from the full native language. | +| Observable side effects | Native DSL implements `print`, restricted to uniform/scalar values. Arbitrary Python I/O is not native DSL. | **High for deterministic behavior.** Ordering and repetition depend on block scheduling, parallelism, partial evaluation, and repeated lazy evaluation. | Exclude `print` and external effects from portable kernels. Do not silently strip them. Host diagnostics remain execution policy. | +| Numeric promotion and conversions | Current DSL supports arithmetic, casts, mixed operands, predicates, and math more broadly than portable 0.1. | **High but essential.** Define intermediate/result types, literal typing, division, casts, predicate rounding, and a bounded supported function list. | Include ordinary real/integer/Boolean numerical use cases; interpreter-first. Do not promise all native numeric combinations automatically. | +| ND index/shape symbols | `_flat_idx`, `_i`, `_n`, `_ndim` already have native synthesis and Python constructor tests. | **Medium-high.** Explicit logical shape, evaluation region, origin/coordinates, slicing/view interpretation, broadcasting and padding rules. | Include a defined ND-context capability; do not expose physical chunk layout as logical semantics. Unsupported host contexts must reject. | +| Fixed-width strings and bytes | Native string locals/returns, width inference, comparisons, concat, transforms, substring and split operations; Python string DSL tests. Interpreter-only today. | **Medium-high.** Typed widths, endianness/code units, constants, first-NUL semantics, truncation, Unicode mapping version and whitespace tables. | Include a bounded fixed-width string/bytes capability. Existing operations are candidates, not all certified yet. | +| Variable-length string representation | Native `me_eval_varlen()` packs evaluated fixed-width values into Arrow-style offsets/data. Intermediates stay fixed-width. | **Medium for an adapter; high for a new language type.** Layout, capacity, offsets and ownership versus dynamically sized intermediates. | Defer a portable varlen ABI/type. Do not reject a fixed-width kernel merely because its host stores the result as varlen. | +| Authoring conveniences | `_NumpyAttrCallRewriter`, `_StringSyntaxRewriter`, `_RowSubscriptRewriter` in `dsl_kernel.py`. | **Low-medium.** Correct normalization, name resolution, evaluation order, typed captures and explicit column mappings. | Include supported normalization: `np.sin(x)`, string methods/membership/split sugar, static `row["col"]`. Dynamic DataFrame operations are not an existing native DSL feature. | +| Persisted graph and storage bindings | `b2objects.py` and `lazyexpr.py` preserve sources and operand references; reconstruction still uses `kernel_from_source`. CTable has DSL-backed column round trips. | **High integration scope.** Kernel portability does not define remote access, authentication, recursive graph execution, or storage identity. | Integrate portable kernel payloads and explicit bindings in the host container. Exclude arbitrary Python-object persistence; do not claim all host graphs/storage protocols are portable. | + +## Agreed exclusion list + +These are the kernel exclusions for the first public release: + +1. **Unsupported reductions inside control-flow bodies and implicit global + collectives.** Permit top-level reductions and supported reduction conditions, + including an `if all(...)` condition within a loop. This is distinct from a + reduction assignment/return inside the body. No implicit reinterpretation of + block reductions as full-array reductions. Classify operations by semantics: + elementwise `fmin`/`fmax` are not block reductions. +2. **Externally registered functions, closures, and runtime object dependencies.** + No function pointers, Python callbacks, opaque object captures, or implicit + module/global state. Supported scalar captures become immutable typed values; + arrays/columns become explicit host bindings rather than hidden captures. +3. **Complex-valued computation.** Defer until there is a portable cross-platform + interpreter contract, independently of JIT availability. +4. **Observable kernel side effects**, including native `print`. Host tracing is + not a persisted kernel effect. +5. **Non-strict FP semantic modes** (`fast`/`contract`) for the initial contract. + Strict arithmetic and explicitly documented math-function accuracy are the + baseline; this does not imply bitwise-identical transcendental results across + all platforms. Persist semantic requirements, not compiler paths or caches. +6. **Dynamically sized intermediate/output types and arbitrary memory access.** + Retain statically bounded string widths and explicit output cardinality + (elementwise or supported block-scalar reduction). + General gathers/scatters, mutation of inputs, arbitrary array subscripting, + and dynamically selected columns are not new 1.0 features. Several of these + are already outside today's DSL; distinguish them from lost capabilities. + +Separate exclusions from the *artifact/host scope*: generated machine code, +Python source execution on import, pickled runtime objects, external function +registration protocols, and universal graph/remote-storage execution. Saving a +portable kernel in a LazyArray does not itself make every referenced data source +available in C or another language. + +## Block-local reduction contract to specify + +The reduction group is the **current evaluation block**, not the full logical +array. Partition-dependent results are permitted; reproducing those results +requires preserving the evaluation partition. The persistence/host integration +must record or unambiguously reconstruct that partition, not silently retile it. +Logical ND coordinates and physical evaluation grouping remain distinct concepts. + +The mandatory motivating case is `if all(escape_iter != limit): break` in the +Mandelbrot loop. It is a block-level early-exit optimization, not an array-wide +synchronization primitive. Conformance must test different partitions and show +both this use case and a deliberately partition-dependent reduction result. + +The approved policies below settle valid-element/active-mask participation, +empty identities, scalar broadcast, partition preservation, and ordered sums. +Before freezing, complete output-cardinality and per-operation details. +Masked/nested conditions that current implementations cannot +define consistently must be identified explicitly, not accidentally admitted. + +## Hard numeric domains: not syntactic exclusions + +Overflow, non-finite/out-of-range integer casts, division by zero, and invalid +shift counts depend on runtime values. They cannot generally be detected by a +save-time validator. Do not claim that successful persistence proves their +absence. The agreed rules below require runtime errors for the specified invalid +integer operations and conversions, and IEEE-style floating behavior. Implement +those checks rather than relying on caller preconditions for these cases. + +Likewise, unsigned integers, narrower widths, and each math builtin need an +explicit type/function matrix. They are open inventory decisions, not covered +by a blanket claim of full miniexpr compatibility. Real arithmetic remains a +major specification task even with limited reductions and no complex numbers. + +## What should not be excluded merely to save JIT work + +- Integer-input division, supported mixed numeric inputs, nested casts and + arithmetic predicates, once their interpreter semantics are defined. +- An agreed set of math builtins and elementwise `where`. +- ND symbols supplied through a defined logical context. +- Fixed-width Unicode/bytes kernels, including bounded string returns. +- Static column access and supported NumPy/string authoring sugar after lowering. + +For example, the existing row rewrite accepts a single row parameter used only +as `row[]` and records parameter-to-column mappings. It is not a +general translation of arbitrary `df[expr]` indexing. + +## Evidence and documentation gaps + +- `../miniexpr/doc/dsl-syntax.md`: statements, ND context, string locals/returns, + registered functions, reductions and print. +- `../miniexpr/doc/strings.md`: UCS4 versus bytes, NUL termination, width bounds, + Unicode truncation, and the existing varlen output adapter. +- `../miniexpr/src/dsl_compile.c`: actual reduction restrictions; existing + unsupported combinations are not an additional portable-only limitation. +- `src/blosc2/dsl_kernel.py`: authoring validator and normalization boundaries. +- `tests/ndarray/test_dsl_kernels.py`, `tests/ndarray/test_string_output.py`, and + `tests/ctable/test_ctable_dsl_columns.py`: ND, string, and persistence use cases. +- `src/blosc2/b2objects.py`, `src/blosc2/lazyexpr.py`: existing persistence paths. + +Some references disagree: Python's DSL page still describes reduction-in-body +behavior as silently incorrect, while current native compilation rejects it; +the strings guide's scalar-condition wording also needs reconciliation with +masked string DSL behavior. Resolve against implementation/tests before freezing +normative text. This inventory is a source review, not new feature certification. + +## Agreed semantic and compatibility decisions + +The following choices were approved after the feature-level scope discussion. +They govern portable 1.0; they are not claims about existing full-DSL behavior. + +### Numeric computation + +- Operand types and explicit casts determine intermediate computation; output + conversion happens last. An output dtype must not change the calculation. + In particular, float32 `(x + 1.0) - x` computes in float32 even when the output + is float64. Apply the new rules through the versioned portable path without + silently redefining ordinary full-DSL execution. +- Literals adopt the operand type where representable; otherwise use explicit + documented promotion rules or reject unrepresentable literals. Mixed typed + operands follow one enumerated promotion table. +- Integer `/` produces float64; homogeneous float32 division stays float32. +- `int(expr)` evaluates its argument first, then truncates toward zero. +- Boolean arithmetic uses numeric zero/one. Explicit `bool()` and Boolean output + perform truth conversion. +- Integer overflow, out-of-range integer narrowing, non-finite/out-of-range + float-to-integer conversion, integer floor-division/remainder by zero, and + invalid shift counts are runtime errors. +- Floating arithmetic retains IEEE-style NaNs, infinities and signed zero, + including floating division by zero. Enumerate math-function domain behavior + and accuracy guarantees separately; strict does not imply universally + bitwise-identical transcendental results. +- JIT must implement the same checks and semantics or fall back before execution. + +### Block reductions + +- Reductions consume valid, currently active lanes of the evaluation block. + Top-level reductions see all valid block elements. Supported nested conditions + see their participating lanes; exclude padding and lanes that have returned + or left the relevant loop. +- `any(empty)` is false, `all(empty)` is true, and `sum(empty)` is zero. + Empty `min`/`max` is a runtime error. Specify other admitted reductions in the + operation matrix before freezing. +- A reduced scalar controls or broadcasts only to participating lanes. +- Persist the logical evaluation partition independently of physical storage. + Rechunking must not silently change reduction groups. Partial reads evaluate + the original groups needed for the slice, then select requested results. + Explicit repartitioning may change results and must be described as such. +- Specify accumulation dtype and logical lane order for floating sums; initially + exclude reassociation into a different reduction tree. Integer accumulation + obeys checked-overflow rules. Relaxed reductions are a later capability. + +### Persistence and legacy formats + +- New DSL-kernel saves normalize authoring syntax/captures, validate portable + 1.0, and persist the artifact, bindings and required execution context. +- Unsupported kernels fail saving with actionable diagnostics; there is no + silent fallback to Python-specific persistence. +- Detect legacy persisted kernel formats and require explicit opt-in before + using them, through the applicable deserialization policy. Never execute + legacy Python reconstruction merely to detect the format. +- **No migration tooling or automatic conversion path is in scope.** Users who + want a portable 1.0 artifact must follow the guidelines, manually adapt their + kernels/bindings, and save through the validated 1.0 path. Legacy opt-in is + permission to use the old representation, not a claim of portable compliance. +- Publication guidance must document supported operations, the numeric behavior + change, reduction grouping, and legacy opt-in/manual adaptation. + +## Remaining specification drafting + +Implementation-led drafting can cover schema fields, capability negotiation, +typed encodings, explicit column bindings, existing ND coordinate conventions, +string width/Unicode rules, diagnostics, and compact conformance scenarios. +Use verified interpreter behavior as evidence, not inconsistent documentation or +an accidental backend behavior as the normative contract. + +The policy choices above are settled. Draft the exact promotion table, literal +representability rules, function/type matrix, reduction NaN behavior and +accumulation dtypes, mask transitions, partition encoding, and legacy format +discriminators/opt-in API against them. Surface contradictions or additional +compatibility consequences for review rather than changing approved policy. + +Draft an explicit dtype/function matrix based on current use cases; surface any +proposed omission of an existing useful operation for review. Pin string mapping +behavior and document truncation rather than silently redesigning it. Do not +promise bitwise-identical transcendental results without an implementation that +can deliver them. + +## Bounded acceptance gates + +1. Draft and review the detailed 1.0 semantics, type/function matrix, capability + requirements, and detailed rules implementing the agreed decisions above. +2. Implement native validation and interpreter conformance for those contracts; + reject unsupported kernels with actionable diagnostics. +3. Integrate portable payloads in kernel persistence, preserving typed bindings, + ND context and reduction partition requirements. Verify standalone C loading + without Python reconstruction, plus host save/load round trips. +4. Certify supported platforms. JIT must preserve the contract or select fallback + before execution; additional acceleration is not a release gate. + +Keep tests orthogonal and retain native ownership of exhaustive language fixtures. +Scope changes require explicit agreement; do not restart a whole-language JIT audit. diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md new file mode 100644 index 000000000..d2deb0e8c --- /dev/null +++ b/plans/portable-DSLs-first-version.md @@ -0,0 +1,841 @@ +# Portable DSL v0.1: first implementation plan + +**Status: conservative 0.1 boundary frozen; release integration in progress.** +The normative specifications are `../miniexpr/doc/dsl-spec/0.1.md` and +`artifact-0.1.md`; the development history below is not the current release scope. + +### Closed scope and release checklist + +- Native source and compiled-tree gates exclude transcendental calls, nested + numeric casts, floating expression arguments to truth casts, mixed computation + intermediates, and unlowered float32/division predicate arithmetic. +- Overflow, out-of-domain casts, zero divisors, and non-default rounding are + explicit runtime-domain exclusions, not promises of wraparound or trapping. +- `frozen-excluded.txt` separates preserved full-language regressions from + conformance. Negative tests verify validation, export, and native import; + unsupported artifacts cannot enter via a bypassed authoring frontend. +- Python package builds enable the optional native artifact adapter by default, + retaining an explicit OFF build option and including yyjson's MIT license. +- Native frozen revision published: `7543a9592410bdfd72f7a7319343dfece55b09e4`. + Python pins that immutable revision, including platform test-harness fixes. +- Stock wheel verification passed without a local native source override: + artifact support defaults ON, yyjson's license is included in wheel metadata, + and interpreter/JIT execution works after moving the build tree. The isolated + wheel passed 2171 focused Python tests; local native and sanitizer suites passed + 431 and 340 tests respectively. Documentation builds with pre-existing warnings. +- Host CI now enables native artifacts on Linux/macOS/Windows and ARM64 jobs. + Local certification is macOS arm64; other platform jobs must be green before + announcing cross-platform release certification. +- Windows conformance certifies the interpreter baseline, not TCC acceleration. +- Numeric scope is closed. Broader math/casts, overflow policy, arbitrary mixed + promotion, and lazy/container integration are later-profile work, not 0.1 gates. + +### Implementation progress + +The first raw-source slice is implemented across Python-Blosc2 and `../miniexpr`: + +- Draft native profile: `../miniexpr/doc/dsl-spec/0.1.md`. +- Native-only Python constructor: `blosc2.DSLKernel.from_source(source)`, with no + Python `exec`, captures, frontend rewriting, or Python-function fallback. +- Shared finite-float64 fixtures and standalone C runner: + `../miniexpr/tests/portable-dsl/`, covering branches, chained comparisons, + bounded loops, and reordered native input bindings. +- Native compiler selection now honors explicit source pragmas ahead of the + `ME_DSL_JIT_COMPILER` environment default, with regression tests. +- Interpreter-only and bundled-TCC build configurations exercise the corpus; + required-JIT tests reject silent fallback. + +The reviewed native baseline is miniexpr revision +`a1f845b590582a85e5fac04184f3f4758e7ca9a5`. Python-Blosc2's existing dependency +pin is unchanged; its installed extension and the authoritative native checkout +are tested independently. Integrating the updated native dependency remains +necessary before claiming the precedence change in distributed Python builds. + +This is **not yet the full v0.1 completion gate**: the wider dtype/semantic audit, +artifact packaging/integration and the remaining semantic audit +are still pending. The raw-source constructor accepts native DSL, not only the +draft portable subset. No new dependency or persisted artifact format has been +introduced. + +Native profile validation is now implemented as `me_validate_portable_dsl()` in +`../miniexpr/src/dsl_portable.c`, with a structured status and optional diagnostic. +It filters native parsed source, checks an explicit homogeneous-input signature, +then compiles with JIT disabled without evaluating a kernel. The corpus runner +uses it before execution. `blosc2.validate_portable_dsl()` exposes the same check +through the extension, with no Python parser/rewriter fallback; older native builds +report `runtime_unsupported`. The local editable environment is rebuilt against +the authoritative checkout via `FETCHCONTENT_SOURCE_DIR_MINIEXPR`; the repository +dependency pin remains unchanged. The draft spec now documents the exact initial +feature filter and distinguishes validation from runtime constraints, backend +compatibility, numerical-accuracy certification, and trust/resource policies. + +Validation for this slice passed: 137 full native tests, 70 conformance/validation +tests with bundled TCC disabled, 24 AddressSanitizer checks, and 210 focused Python +tests with standalone native-runner comparisons enabled. Ruff and diff whitespace +checks passed; the native builds emitted no new compiler warnings. + +P3 now has a candidate artifact schema in `../miniexpr/doc/dsl-spec/artifact-0.1.md` +and a separate optional `miniexpr_artifact` C adapter. With user approval, it uses +yyjson 0.12.0 pinned to `8b4a38dc994a110abaec8a400615567bd996105f`; the raw compiler +does not fetch/link yyjson when `MINIEXPR_BUILD_ARTIFACT` is OFF (the default). +The adapter validates strict JSON structure and duplicate keys, versions and +capabilities, typed scalar encodings, entry point and exact binding coverage, +and portable-profile compilation. It owns source/names/scalars, binds runtime +buffers by name, broadcasts constants in bounded tiles, and separates load, +binding, native evaluation, and allocation failures. Required strict FP semantics +are made explicit on an internal source copy when no FP pragma is supplied; +retained compiler pragmas remain preferences rather than compatibility requirements. +The hand-authored affine artifact runs in the standalone C runner without Python. +The schema/profile remain experimental; native adapter packaging and the remaining +semantic audit are still pending. P4 integration is described below. + +P3 validation passed: 143 full native tests, 75 conformance/validation/artifact +checks with bundled TCC disabled, 29 AddressSanitizer checks, and 210 focused +Python regression tests. A fresh static-only raw-compiler build without artifact, +SLEEF, TCC, or Accelerate dependencies passed 70 conformance/validation checks; +yyjson was not fetched. The adapter and tests also passed `-Wall -Wextra -Werror` +syntax checks. A test-only deprecated `sprintf` warning was fixed; final native +builds emitted no new compiler warnings. + +P4 now exposes `DSLKernel.export()` and `PortableKernel.from_json()` / +`evaluate()` in Python. Export uses the unspecialized author's DSL source and +snapshots exact supported scalar globals/closure values into collision-free +parameters and typed constants. Explicit parameter constants and capture dtype +overrides are supported; no user conversion hooks or kernel calls are used. +Input/output dtypes are explicit and portable native validation plus the C loader +validate every exported artifact. Import calls that same loader through an owned +Cython handle, without Python source/JSON parsing or frontend preparation. +Runtime input names/dtypes/shapes are checked, host storage is adapted, and native +constant broadcast/evaluation is reused. NumPy-alias rewriting now excludes local +and closure shadowing so export cannot silently reinterpret a local attribute call. +The extension links the adapter only if the optional native target is available; +disabled/older builds report unsupported artifact operations. Local builds use +the source override and `MINIEXPR_BUILD_ARTIFACT=ON`; the repository dependency pin +is unchanged because native dependency publication/integration is still pending. +Artifact execution is currently eager NumPy output, not lazyudf/container integration. + +P4 validation passed: 240 focused Python tests with the adapter enabled, including +Python export -> standalone C execution, fresh-process import without the author, +typed scalar boundaries/NaN payloads, snapshot/collision/closure handling, storage +adaptation, native diagnostics, and concurrent interpreter/TCC evaluations. +An actual adapter-disabled editable build passed 212 tests with 28 artifact-only +cases skipped and verified `NotImplementedError` without a Python fallback. +The environment was restored to the adapter-enabled local build. The native +conformance/profile/artifact selection passed 99 checks; Ruff, the standalone +example, and whitespace checks also passed. + +The next conformance slice expands the shared corpus with explicit +input/output dtypes and expected compile/evaluation outcomes. It covers all five +candidate dtypes, exact int64 boundaries and comparisons above `2**53`, signed +zero and non-finite identity, Boolean-result semantics, bounded loop control, +small finite casts, and sampled `sin` results. Integer fixtures are decoded and +compared without a floating-point intermediate. CTest also generates CC variants +that change only the compiler preference, uses build-local JIT caches, and applies +timeouts. Missing-return kernels now have runtime JIT support: successful paths, +mixed successful/failing elements, loop returns, and hybrid vector cleanup are +tested under required-JIT policies. Semantic missing-return errors propagate +without interpreter retry; generic backend failures retain existing fallback. +This advances P2 coverage but does not freeze +overflow, general promotions/conversions, or global transcendental accuracy rules. + +The numeric stabilization slice adds 25 shared exact output-conversion fixtures +(the full supported 5×5 dtype matrix), including representable cast boundaries, +int64 rounding, signed zero, subnormals, and non-finite identity/truth values. +Native integer-literal validation now checks normalized digits exactly so +`2**53 + 1` cannot bypass the profile limit through double rounding. +The audit found unresolved integer-valued division promotion and float32 math +intermediate/condition discrepancies across JIT and linked interpreter paths. +Shared reproducers and strict expected-failure artifact tests preserve the gaps; +they block freezing rather than redefining portable semantics. Details are in +`../miniexpr/doc/dsl-spec/numeric-audit-0.1.md`. Native publication, dependency +integration, and the rest of the semantic matrix remain pending. + +This slice passed 328 focused Python tests with 10 strict expected JIT failures, +202 native portable/profile checks in the TCC-enabled build, and 55 selected +interpreter-only portable/profile/artifact checks. Ruff and whitespace checks +passed, and the native builds emitted no new warnings. + +The next slice corrects the first division-promotion cases using C arithmetic +rendered from the native compiled expression tree. Seven exact shared fixtures +cover leaf casts, literal folding, locals, loop indices, float32 rounding, and +branch conditions under interpreter/TCC/CC execution. Typed text participates in +the IR fingerprint and code-generation cache version 14. Text-only hybrid plans +cannot bypass the correction. Unsupported typed contexts conservatively retain +interpreter execution; native-only artifact import and compiler precedence are +unchanged. The six division expected failures are removed; four float32 math +expected failures remain. Broader nested-cast probes exposed additional native +evaluator dispatch anomalies and an unisolated abort, recorded in the numeric +audit as separate blockers rather than promoted conformance claims. + +Division-slice validation passed 377 focused Python tests with four remaining +math expected failures, all 272 native tests in the TCC-enabled build, and 61 +selected interpreter-only portable/profile/artifact/IR/codegen tests. Required-JIT +division fixtures passed under both TCC and CC. Ruff and whitespace checks passed; +native builds emitted no new warnings. At that stage the nested-cast reproducer +was excluded from execution conformance pending a native evaluator investigation. + +The following memory-safety slice isolated that abort with AddressSanitizer: +nested conversion code selected a float64 writer for a float32-sized scratch +buffer. The writer now matches the actual typed evaluator's output representation +while retaining source-dtype evaluation. Exact interpreter fixtures and native/ +Python tests cover float32/float64 input/output pairs, counts 1/5/257, repeated +evaluation, and requested-JIT fallback. Sanitizer validation also corrected an +existing mixed-type test's undersized output allocation (int32 + float32 infers +float64). All 217 sanitizer-build tests, all 274 regular native tests, and 401 +focused Python tests passed, with four existing math expected failures. Ruff and +whitespace checks passed; builds emitted no new warnings. This fixes the confirmed +buffer overflow, not broader nested-cast promotion or float32 math semantics; +those still block freezing 0.1. The native dependency pin remains unchanged. + +The next slice corrects `int()`/`bool()` argument value semantics: evaluate in the +argument's compiled dtype before truncation/nonzero truth, instead of promoting +variables or recomputing arithmetic in the cast result dtype. Integer casts +preserve int64 values beyond `2**53` without a double round trip; Boolean casts +retain non-finite and subnormal truth. Five exact shared fixtures require real +TCC/CC execution as well as interpreter agreement; a separate division audit +case checks conservative interpreter fallback. Native repeated-evaluation tests +cover counts 1/5/257 and floating widths, while Python tests cover all five output +dtypes for value casts. All 295 regular native tests, 233 AddressSanitizer tests, +and 484 focused Python tests passed, with four existing math expected failures. +No new build warnings were emitted. General `float()`/division promotion, +out-of-domain integer casts, and float32 math semantics remain unfinished; the +profile is still experimental and the dependency pin remains unchanged. + +The following float32 math slice resolves the two remaining expected-failure +reproducers for leaf `sin` widening and `cos` branch conditions. Strict leaf +float32 sin/cos uses scalar float math at every count, avoiding the one-ULP SIMD +variation observed at count 257; comparison evaluation no longer pre-promotes +variables before consuming operand values. Owned typed JIT math text preserves +float rounding before widening/comparison and bypasses text-only hybrid plans; +it enters the IR fingerprint, with code-generation cache version 15. The two +audit fixtures are promoted to interpreter/TCC/CC conformance, with a local-value +widening fixture and native/Python repeated-execution checks at counts 1/2/257. +All 307 regular native tests, 242 AddressSanitizer tests, and 525 focused Python +tests passed without expected failures or new build warnings. General nested +math, other functions, inexact comparison constants, broader promotion/cast +domains, and cross-platform math accuracy still block freezing the profile; +dependency integration remains pending. + +The next literal/promotion slice corrects pure float32 arithmetic: C double +literals must not bypass native operand or intermediate rounding for `+`, `-`, +`*`, and unary negation. Five shared exact fixtures cover exact/decimal literal +rounding, subtraction, locals, and differing floating output context. Literal +typing is contextual: a float64 requested output can make literal-containing +arithmetic compute in float64, rather than just widen a float32 result. The new +owned lowering targets float32 computation only, bypasses text-only hybrid plans, +enters the IR fingerprint, and uses code-generation cache version 16. Ordinary +float64 hybrid branch optimizations are preserved. Native/Python tests check +counts 1/5/257, repeated execution, both floating output widths, and scalar +addition/subtraction/multiplication. All 328 regular native tests, 258 +AddressSanitizer tests, and 632 focused Python tests passed without expected +failures or new build warnings. Calls/casts, mixed intermediate dtypes, integral/ +Boolean arithmetic, overflow, and the complete promotion contract remain audit +work; the profile is still experimental and dependency integration is pending. + +The while-loop execution-policy slice fixes missing JIT cap enforcement. Scoped +body-entry counters preserve exact-limit exits, `continue`, nested/re-entered +loops, and condition-prefix lowering. The normalized host cap enters the IR +fingerprint, with code-generation cache version 17; a changed policy after +compilation bypasses both JIT evaluation paths for current-policy interpreter +execution. Semantic cap errors propagate without interpreter retry and release +hybrid temporaries. Four shared fixtures and expanded native/Python tests cover +small caps, disabled/invalid policies, counts 1/5/257, repeated/empty execution, +cache isolation, policy changes, and boundary control flow. A separate +interpreter mixed-lane chained-while discrepancy is preserved in +`audit/while_cap_chain` and a strict expected-failure test; it blocks freezing +the profile alongside the remaining numeric domains and dependency integration. +Validation passed all 344 regular native tests, 270 AddressSanitizer tests, and +727 focused Python tests with one strict chained-while expected failure. Ruff +and whitespace checks passed; native builds emitted no new warnings. + +The subsequent masked-local correction resolves that chained-while discrepancy. +Scalar chain operands now read the active lane of local buffers rather than +assuming compile-time uniformity survives masked updates. Root reduction copies +broadcast their scalar results into full-width DSL destinations, preserving +existing nonportable reduction behavior. `while_cap_chain` is promoted to the +passing corpus, with numeric/Boolean masked-local fixtures and expanded lane, +dtype, count, and repeated-evaluation matrices. Validation passed 357 regular +native tests, 280 AddressSanitizer tests, and 895 focused Python tests without +expected failures or new native build warnings. The remaining numeric contracts +and publication/integration gates are unchanged. + +The next numeric slice fixes float32 rounding lost around same-dtype `float()` +calls: `(float(x) + 1.0) - float(x)` and `float(x + 1.0) - float(x)` now return +the interpreter's zero at `x = 2**24`, instead of JIT's previous one. Typed pure +arithmetic admits only float32-result calls with float32-computed arguments, +recursing through the already supported arithmetic. Root calls, nested/repeated +calls, and local assignments preserve rounding; float64 output retains observed +contextual literal semantics. Mixed/integral casts and general calls are not +certified. Four shared fixtures and expanded scalar/vector matrices cover both +compilers, output widths, decimal literals, signed zero, subnormals, extrema, +infinities, and NaNs. Code-generation cache version is 18. Validation passed all +373 regular native tests, 292 AddressSanitizer tests, and 1019 focused Python +tests without expected failures or new native build warnings. + +The Boolean-operand floating-arithmetic slice corrects interpreter storage: +`bool(x) + bool(x)` with integral inputs and floating output now returns two +for nonzero input rather than writing an invalid Boolean byte before conversion. +The DSL preserves the requested floating computation dtype for Boolean-inferred +arithmetic, while casts still consume their arguments in their own dtype. +Typed unary negation casts before operating so false produces floating negative +zero, including top-level bool-assigned locals. Float64 lowering stays narrow +to preserve hybrid branch/select plans; cache version is 19. Five shared fixtures +and direct/local scalar/vector matrices passed 393 regular native tests, +307 AddressSanitizer tests, and 2014 focused Python tests without expected +failures or new native build warnings. `audit/bool_output_fraction` preserves an +unresolved Boolean-output arithmetic discrepancy, not a normative rule; that +domain and broader mixed-type promotion remain publication gates. + +The next slice implements the user-selected numeric-then-truth Boolean contract. +Boolean operands are numeric zero/one, fractional literals and intermediates +stay numeric, and truth conversion occurs only at a Boolean output or explicit +`bool()` call. Boolean-only/integral-literal contexts compute in int64; +fractional-literal contexts compute in float64. Locals and cast arguments retain +numeric values, and a `2**53` cancellation regression checks exact integral +intermediates. `bool_output_fraction` is promoted from the audit corpus, with +four additional shared fixtures and scalar/vector backend matrices. Cache +version is 20. Validation passed 413 regular native tests, 322 AddressSanitizer +tests, and 2161 focused Python tests without expected failures or new native +compiler warnings. This closes the sampled Boolean-output discrepancy, not the +remaining arbitrary mixed-type promotion and overflow contracts. + +## 1. Goal and scope + +Establish a small, versioned miniexpr kernel language that executes independently +of Python, then demonstrate Python export and standalone C import of a portable +kernel artifact. Deliver this vertical slice before deciding how far to extend +the language, its packaging, or other language bindings. + +This implements the first stage of [the broader proposal](portable-DSLs.md), with +two refinements: + +- Native execution and cross-language conformance come before artifact design. +- Compiler pragmas remain in portable source and override local compiler defaults, + while retaining best-effort fallback. + +Python-like syntax is compatible with a language-independent standard. The +contract is a documented grammar and semantics, implemented without Python, not +the absence of syntax familiar to Python users. + +### Two independently useful deliverables + +1. **Portable source:** identical source plus an explicit signature can execute + through miniexpr's C API and Python-Blosc2. +2. **Portable artifact:** a small manifest carries that source, signature, typed + constants, and semantic requirements between environments. + +The first deliverable must not depend on completing the second. + +### Initial required scope + +- One named kernel, scalar-per-element execution, one scalar result per element. +- An explicit ordered input signature and output dtype. +- A documented subset of existing native numeric and Boolean syntax. +- Native interpreter execution as the mandatory baseline. +- TCC execution where supported, with tests proving which backend actually ran. +- Raw-source construction from Python without reconstructing a Python function. +- Python export of a kernel with a captured numeric scalar constant. +- A standalone C loader that consumes the exported artifact without Python-side + preparation at load time. +- Version checks, typed binding validation, and useful unsupported-feature errors. + +### Deferred beyond the initial completion gate + +- Complete string/bytes support and normalization of every authoring convenience. +- New method syntax, membership syntax, or split unpacking in miniexpr. +- Reduction kernels, index symbols, arbitrary rank constraints, and broadcasting. +- Complex, structured, object, datetime, nullable, and variable-length dtypes. +- External C functions, closures, and host callbacks in portable artifacts. +- `print` and other observable block-level side effects. +- Whole lazy computation graphs, array storage references, and remote access. +- Integration into Blosc2 persistence containers and generated-code caches. +- Mandatory JavaScript, Rust, or Julia implementations. + +Existing functionality outside this subset continues to work through existing +APIs; it simply is not certified by the initial portable profile. + +### Baseline after reviewing the updated miniexpr checkout + +The authoritative checkout is `../miniexpr`. Its existing `doc/dsl-syntax.md` +is already titled **miniexpr DSL Syntax (Canonical Reference)** and documents +native syntax and several exact semantic/error rules. `doc/dsl-usage.md` already +demonstrates standalone C compilation/evaluation and documents JIT controls. +This work standardizes a portable subset of that language; it does not introduce +native DSL execution from scratch or replace the existing reference. + +In particular, the full language already supports reductions, reserved ND symbols, +`print`, and string-valued kernels. Those features need explicit profile exclusions +in v0.1 rather than descriptions implying they are absent from miniexpr. + +## 2. Ownership and implementation boundaries + +### miniexpr owns the portable language + +Keep the normative specification, native validation, interpreter behavior, and +language conformance corpus with miniexpr. Its implementation is the initial +implementation of the specification, not an implicit substitute for one. + +Keep the kernel DSL distinct from miniexpr's classic expression API. Do not +silently expand or change that API when implementing portable kernel support. + +### Documentation layout and versioning + +Use these paths in the authoritative miniexpr repository: + +- `doc/dsl-syntax.md`: retain the current canonical reference for the full native + DSL. Extend or clarify it where the semantic audit finds omissions; preserve + its existing role and links from other documentation. +- `doc/dsl-usage.md`: retain the practical C API and runtime configuration guide. +- `doc/dsl-spec/0.1.md`: add the normative **portable profile** specification, + identifying the admitted grammar, dtype/function matrix, semantic requirements, + and explicit exclusions from the full DSL. +- `doc/dsl-spec/artifact-0.1.md`: add the versioned manifest and typed-binding + specification once the portable-source milestone works. + +During drafting, derive rules from the canonical reference rather than inventing +a second grammar. At publication, make the versioned profile's normative rules +self-contained or pin incorporated material to an immutable miniexpr revision. +A moving `doc/dsl-syntax.md` link must not silently change the meaning of a +published v0.1 artifact. Link from the current canonical reference to the profile +and explain full-language versus portable-profile support. + +Native work must be made in the authoritative miniexpr checkout and integrated +through the dependency mechanism used by Python-Blosc2. Build-tree copies under +`build*/_deps/` are inspection/build artifacts, not the source of record. + +### Python-Blosc2 owns authoring and integration + +Use `src/blosc2/dsl_kernel.py` as the integration point for current kernel +authoring, normalization, and capture handling. Inspect the existing extension +bridge before adding new native entry points. If artifact handling merits its +own module, use a non-underscored name such as `dsl_portable.py`. + +Separate the native specification from Python guidance in +`doc/reference/dsl_syntax.md`. Link to the normative miniexpr reference and clearly +identify decorator behavior and frontend-only conveniences. + +### Keep the layers independently testable + +1. Parse and validate native source against the portable profile. +2. Validate signature and typed bindings. +3. Check target semantic capabilities. +4. Compile or interpret using the selected execution preference. +5. Execute on caller-supplied buffers. + +Artifact decoding should feed these same operations. It should not introduce a +second language validator or execute frontend rewriting at import time. + +## 3. Define the initial language profile + +Use separate identifiers for the language version and artifact schema version. +Suggested initial values are `0.1` for each; these are experimental compatibility +identifiers, not miniexpr package versions. + +### Candidate core, to freeze after the native audit + +- One top-level `def`, positional named parameters, indentation-based blocks. +- Local assignments, supported compound assignments, and `return`. +- `if`/`elif`/`else`, `for ... in range(...)`, bounded test cases for `while`, + `break`, `continue`, and `pass`. +- Native numeric literals, arithmetic, comparisons, chained comparisons, and + Boolean operators. +- A short, explicit whitelist of native casts and mathematical functions. +- Comments, docstrings, semicolons, and multiline syntax already accepted natively. +- Initially target `bool`, `int32`, `int64`, `float32`, and `float64`; include only + types and operations whose behavior is specified and verified across the + interpreter and supported JIT paths. Do not claim unsigned or mixed-type support + merely because a host API accepts the dtype. + +Start the first executable fixture with `float64`; expand to the frozen core +before declaring v0.1 complete. Record the final operator/function/type matrix in +the normative specification rather than relying on this candidate list. + +### Semantic decisions required before freezing the profile + +Document and test: + +- Parameter binding, duplicate names, local assignment and type inference rules. +- Literal typing, promotions, return conversion, and explicit casts. +- Integer overflow, narrowing, signed division, remainder, and division by zero. +- Floating-point rounding assumptions, non-finite values, and signed zero. +- Evaluation order, single evaluation of chained operands, and short-circuiting. +- Boolean-result rules for `and`/`or`, without implying Python operand-return rules. +- Loop bounds, step handling, and errors such as a zero `range` step. +- Missing returns and use of uninitialized locals. +- The scalar-per-element execution model and independence from chunk boundaries. +- Error categories and when errors occur: validation, compilation, or execution. + +Carry forward the rules already documented in the native reference, verifying +them rather than treating them as undecided design choices: + +- Native parameters match variable names by set membership; caller variable order + may differ from source parameter order. +- Incompatible local reassignment and inconsistent return dtypes are compile-time + errors; an executed path without a return fails at runtime. +- A zero `range` step and exceeding the `while` iteration cap are runtime errors. + Specify the cap and its configuration/portability implications for this profile. +- `//=` lowers to `floor(a / b)`; do not assume Python integer floor-division + precision without auditing intermediate typing. +- Reserved ND names remain reserved even though index-aware kernels are excluded. +- The full DSL permits reductions at top level and in conditions, but not inside + `if`/`for`/`while` bodies. The v0.1 elementwise profile excludes reductions in + all positions, including conditions, to retain chunk-independent semantics. + +Audit existing behavior before promising a rule. If interpreter and JIT disagree, +either fix the discrepancy with regression coverage or exclude that operation/type +combination from v0.1. Do not standardize accidental C undefined behavior. + +Use a small core capability identifier rather than a capability for every token. +Add separate capabilities only for meaningful optional semantic features. Backend +names such as `tcc` are execution preferences, not language capabilities. + +## 4. Compiler preferences and interpreter baseline + +Miniexpr includes an interpreter. It also builds bundled libtcc by default on +supported targets, so ordinary users do not need a separately installed compiler. +TCC may be disabled in custom builds or unavailable on a target; Python-Blosc2 +currently disables it on Windows ARM64. + +For this implementation: + +1. Preserve `# me:compiler=tcc|cc` in portable source. +2. An explicit source pragma wins over the local compiler-selection default. +3. With no pragma, use the runtime's existing default selection policy. +4. Preserve existing best-effort fallback when the preferred backend is unavailable + or cannot compile the kernel. Use the interpreter when it satisfies the required + semantics. +5. Keep ordinary fallback quiet, while making the actual execution backend + observable to diagnostics and conformance tests. + +Audit the current selection path and codify its fallback order; do not invent a +new fallback chain as a side effect of artifact support. A deliberate local +interpreter-only execution mode remains useful for testing and deployments; it +is distinct from choosing a default compiler. + +Reuse the documented `ME_JIT_DEFAULT`, `ME_JIT_ON`, and `ME_JIT_OFF` policies, +including `me_eval_params.jit_mode`, rather than adding a parallel JIT toggle. +Test that explicit `ME_JIT_OFF` prevents JIT even with a compiler pragma: choosing +whether to JIT and choosing which compiler to prefer are separate controls. + +Floating-point requirements are independent of backend preferences. Define how +`# me:fp=strict|contract|fast` maps to the artifact's semantic requirements and +which interpreter/JIT modes satisfy them. Reject conflicting declarations rather +than silently allowing one to override the other. Do not promise identical bits +across all mathematical libraries or imply a stronger meaning of `strict` than +the specification defines. + +Compiler paths, cache directories, and diagnostic settings remain local. The +artifact contains a compiler preference, never instructions to locate or load an +arbitrary executable or library. + +## 5. Raw-source execution before packaging + +Build the first vertical slice around source such as: + +```text +# me:compiler=tcc +def affine(x, scale, offset): + y = x * scale + offset + if y < 0: + return 0.0 + return y +``` + +At this stage all three names can be ordinary explicitly typed inputs. Execute +the exact same source through a standalone C program and Python, using the same +input values and declared output type. + +Expose or reuse a Python raw-source construction path that accepts source and a +signature. Loading it must not use Python `exec`, fabricate a Python function, or +inspect globals. Final public names should follow the existing kernel API after +inspection; do not add competing constructors unnecessarily. + +The C executable must link miniexpr without libpython. Python may launch it as a +test subprocess, but the executable must also run directly from a shell with +prepared fixtures. Native execution requires no Python-generated temporary code. + +Start from the existing `me_compile()` / `me_eval()` / `me_free()` flow documented +in `doc/dsl-usage.md`; use `me_compile_nd_jit()` where its explicit JIT controls +are needed. Audit existing parser diagnostics and backend test helpers before +introducing any new native API. The new work here is a reusable conformance runner +and profile validation, not a replacement execution engine. + +## 6. Minimal artifact design + +After raw-source conformance works, use a standalone UTF-8 JSON manifest as the +first interchange format. Keep it small enough for a native loader and independent +of Blosc2 storage. Reassess container integration after v0.1 works end to end. + +### Proposed fields + +| Field | Meaning | +| --- | --- | +| `schema_version` | Artifact encoding/structure version | +| `language` | Language identifier and version | +| `requires` | Required semantic capability identifiers | +| `source` | Complete canonical native DSL source, including retained pragmas | +| `entry_point` | Name of the single kernel in the source | +| `inputs` | Ordered runtime input names and language-neutral dtypes | +| `constants` | Explicit named, typed scalar bindings | +| `output` | Output dtype and scalar-per-element execution contract | +| `semantics` | Required floating-point profile and other supported requirements | +| `metadata` | Optional non-semantic producer information | + +Freeze required fields and unknown-field handling in a small schema document. +Reject duplicate JSON keys, duplicate bindings, and unknown semantic fields. +Permit explicitly designated informational metadata without interpreting it. +Do not introduce a runtime schema-validation dependency just for this manifest. + +### Signature and constants + +Use the source's parameter list as the full native parameter order. Every +parameter must be supplied exactly once by either `inputs` or `constants`. +Disallow overlapping names, missing bindings, and unused binding entries. +`inputs` follows source order with constant parameters omitted. + +This ordering is an artifact convention, not a new native-language restriction. +The adapter must map names to the `me_variable` array and supply evaluation +pointers in that array's order, as the current native API requires. Include a +reordered-variable fixture to prevent accidental positional-only binding. + +For captured constants, export a collision-free explicit parameter and record its +typed value in `constants`. This avoids reliance on implicit native globals or +Python literal formatting. Reuse native scalar binding support if available; +otherwise provide an adapter with defined broadcast and lifetime behavior rather +than exposing an array-sized copy requirement in the artifact contract. + +### Typed scalar encoding + +- Boolean: JSON Boolean with dtype `bool`. +- Integer: decimal string with explicit fixed-width dtype and range validation. +- Float: exact IEEE-754 bit-pattern hex string with explicit `float32` or + `float64` dtype and fixed digit count, written most-significant byte first. +- Decode floats into the host representation explicitly; do not rely on native + endianness or JSON numeric precision. + +This preserves large integers, signed zero, infinities, and NaNs in transport. +Transporting a NaN bit pattern does not promise that arithmetic preserves its +payload. Strings/bytes get a separate encoding decision when that profile is added. + +### Compatibility + +For the experimental implementation, accept only explicitly supported language +and schema versions. Reject unknown versions clearly instead of guessing forward +compatibility. Keep frozen example artifacts as regression fixtures. Once a v0.1 +contract is published, change semantics under a new version rather than silently +reinterpreting existing artifacts. + +## 7. Python export and import + +### Export pipeline + +1. Obtain the existing author's source and explicit specialization signature. +2. Normalize supported frontend conveniences using the existing frontend machinery. +3. Discover scalar captures without calling the kernel or arbitrary user callbacks. +4. Resolve each supported capture to an explicit dtype and snapshot its value at + export time. Require an explicit type when inference would be ambiguous. +5. Replace captures with collision-free bound parameters. +6. Run native portable-profile validation on the resulting source and signature. +7. Derive required capabilities and emit the manifest deterministically. + +Do not treat `kernel.dsl_source` alone as the finished artifact. Audit whether +existing normalization or scalar specialization has already discarded type or +capture information; retain that information before lowering if necessary. + +The first required example captures a numeric scale constant. After export, +changing the Python global must not change execution of the saved artifact. +Reject unresolved names, arrays/objects as captures, external functions, and +unsupported constructs with the relevant name and source location where possible. + +### Import pipeline + +Decode and validate the artifact, construct the typed bindings, then use the same +native source path as C. Import must not require the original module, function, +decorators, globals, or frontend rewriters. + +Provide a raw-source portable validator first. An optional strict authoring mode +that rejects frontend sugar and implicit captures can build on it later; it need +not block the first export/import milestone. + +## 8. Native artifact loader + +Implement the loader as a thin adapter adjacent to miniexpr's native API, with +clear ownership of decoded source, binding buffers, compiled handles, and errors. +The language compiler must remain usable without JSON artifact support. + +Before choosing JSON machinery, inspect existing dependency policy and available +parsers. Reuse an appropriate existing facility where possible; any new dependency +requires an explicit decision. Avoid a hand-written partial JSON parser that only +accepts the Python exporter's preferred formatting. + +Loader stages: + +1. Decode JSON with bounded lengths and validate field structure. +2. Check schema/language versions and required capabilities. +3. Decode and range-check typed constants and signature information. +4. Validate entry point and exact binding coverage against native source. +5. Check semantic requirements independently of the compiler preference. +6. Construct an executable handle using the existing interpreter/JIT machinery. +7. Bind caller-provided data and execute; free all owned resources on every path. + +Define logical buffers as same-length inputs and output for the initial profile, +with scalar constants broadcast to each element. Specify empty-input behavior, +host dtype conversion, buffer ownership, and binding lifetime. Keep physical +strides, endian conversion, and storage layout in adapters rather than in source. + +Distinguish malformed artifacts, unsupported requirements, invalid source, +binding errors, and evaluation errors. Lack of the preferred compiler alone is +not an invalid artifact. + +The loader does not provide a sandbox. It must not execute Python, follow artifact +paths to load libraries, or use artifact data as shell commands. General resource +isolation remains a host responsibility rather than a new v0.1 subsystem. + +## 9. Conformance corpus and verification + +Create language-neutral fixtures containing source, signature, inputs, expected +outputs or error category, and numeric comparison rules. Keep native language +fixtures authoritative in miniexpr and consume the same cases from Python tests; +avoid independently maintained copies with diverging expectations. + +Seed the corpus from existing native tests, especially `test_dsl_syntax.c`, +`test_dsl_python_syntax.c`, `test_dsl_literals.c`, `test_dsl_comparisons.c`, +`test_dsl_bool_locals.c`, `test_dsl_guards.c`, and `test_dsl_jit_options.c`. +Reuse their helpers where appropriate and preserve implementation-level coverage; +only promote profile-relevant cases into the shared language-neutral corpus. + +Required groups: + +- Basic arithmetic, branches, loops, casts, and every admitted dtype/function. +- Chained comparisons, short-circuit guards, and operand evaluation order/count. +- Integer boundaries, promotions, narrowing, and defined arithmetic errors. +- Floating-point boundaries, signed zero, NaNs, infinities, and profile-specific + tolerances for mathematical functions. +- Signature ordering, captured constants, name collisions, and empty buffers. +- Invalid syntax, missing returns/bindings, unsupported types and capabilities. +- Malformed JSON, duplicate fields, invalid scalar encodings, unknown versions, + and inconsistent semantic declarations. +- Compiler pragma precedence, no-pragma defaults, and interpreter fallback. +- Frozen artifact import without its originating Python module. + +Use expected values derived from the specification, not only agreement between +two implementations. For single-evaluation tests, use native test instrumentation +where pure numeric outputs cannot reveal repeated evaluation; do not require +portable user callbacks just to make the tests observable. + +Run all core cases explicitly through the interpreter. Run the applicable corpus +through TCC, and through CC where available. JIT tests must inspect the actual +backend or use an existing no-fallback test mode so an interpreter fallback cannot +masquerade as JIT conformance. Fallback itself has separate tests. + +Include at least one configuration with TCC disabled and one normal bundled-TCC +configuration. Exercise representative supported platforms in CI as available. +JavaScript is a later capability-specific target, not a first-version gate. + +Python, build, and test commands in this repository must use the `blosc2` conda +environment. Run focused native tests and Python DSL tests as each layer changes, +then the repository's required checks before completion. Keep unrelated tests and +benchmarks outside the implementation loop unless failures justify investigation. + +## 10. Ordered work packages and completion gates + +### P0 — Audit and freeze a small contract + +- Locate authoritative miniexpr sources and dependency integration points. +- Record the reviewed miniexpr revision and use its existing canonical reference, + usage guide, and native DSL tests as the specification/conformance baseline. +- Inventory native parsing, dtype rules, execution APIs, capture specialization, + scalar bindings, and backend selection. +- Write the initial normative grammar/semantics and supported operation matrix. +- Record exclusions and resolve interpreter/JIT discrepancies for admitted cases. + +**Gate:** a precise candidate profile and first fixtures, with no dependence on +Python behavior to explain native semantics. + +### P1 — Execute identical raw source in C and Python + +- Add the standalone C runner and Python raw-source integration. +- Execute the affine example and control-flow cases with explicit signatures. +- Add interpreter/TCC backend observability and selection-policy tests. + +**Gate:** the same source executes without Python preparation in C and through +Python, and interpreter execution is verified explicitly. + +### P2 — Establish the shared core conformance suite + +- Expand fixtures across the frozen core syntax and dtype matrix. +- Document numerical comparison rules and runtime error categories. +- Verify interpreter/TCC parity and applicable CC coverage. + +**Gate:** all claimed v0.1 core semantics have representative positive, boundary, +and rejection coverage. The portable-source milestone is independently usable. + +### P3 — Implement the minimal artifact and C loader + +- Freeze the initial manifest and scalar encodings. +- Implement native decoding, binding validation, ownership, and error handling. +- Load a hand-authored artifact and reuse the P1 execution path. + +**Gate:** a standalone C executable loads and runs a fixture artifact without +Python and rejects malformed or unsupported artifacts explicitly. + +### P4 — Export and re-import a Python-authored kernel + +- Implement explicit-signature export and captured scalar parameterization. +- Implement Python import through the native portable-source path. +- Export in Python, execute in C, and compare against specified expected results. +- Verify snapshot semantics, deterministic export, and independence from the + original Python module. + +**Gate:** the complete Python-to-C artifact round trip works for the numeric core. + +### P5 — Document, stabilize, and review the first version + +- Publish the native reference and short Python/C examples. +- Document pragma precedence, fallback, version handling, and supported dtypes. +- Preserve frozen artifacts and complete focused regression/platform checks. +- Record practical limitations and review the next priorities with the user. + +**Gate:** v0.1 meets the checklist below. Further expansion is a separate decision. + +## 11. Follow-up candidates after v0.1 + +Extend the same corpus and manifest rather than redesigning the architecture. + +1. **Strings and bytes:** define encodings, lengths, case behavior, substring rules, + output storage, and dtype descriptions before claiming portable-profile support. + Start from existing `doc/strings.md` and `doc/dsl-syntax.md`: native string + locals and returns already work, widths have documented constraints, shorter + returns are NUL-padded, and string kernels execute on the interpreter rather + than the JIT. A string profile must not require TCC support to be portable. +2. **Native string methods:** consider a fixed whitelist lowering to existing + functions, without general object dispatch. +3. **Native membership:** consider `in`/`not in` for explicitly supported + string/bytes operands, with defined evaluation order. +4. **Frontend adapters:** NumPy aliases and pandas row access remain authoring and + binding conveniences; add portable column mappings when needed. +5. **Split unpacking:** decide missing-separator, empty-separator, result-count, + and evaluation-count semantics before lowering to `split_part` calls. Two + extraction calls do not automatically reproduce Python unpacking behavior. +6. **Additional execution profiles:** reductions, index-aware kernels, other + backends, and foreign-language bindings, each with matching conformance cases. + +Promotion of useful syntax into miniexpr is welcome when it benefits authors in +multiple environments. Eliminating a Python rewrite alone is not a sufficient +reason to enlarge the language. + +## 12. First-version acceptance checklist + +- [ ] Native versioned specification defines the admitted language subset. +- [ ] Same raw source and signature execute from C and Python. +- [ ] Interpreter passes the full core conformance corpus. +- [ ] Supported JIT paths pass applicable cases with actual backend verification. +- [ ] Compiler pragmas survive export and override local compiler defaults. +- [ ] Best-effort fallback preserves required semantics and remains quiet by default. +- [ ] Explicit typed constants survive a Python export/C import round trip. +- [ ] Saved artifacts execute without the original Python module or Python runtime. +- [ ] Loaders reject unsupported versions/capabilities and malformed bindings. +- [ ] Dtype, numeric, error, and buffer contracts are documented and tested. +- [ ] Existing Python authoring and evaluation APIs retain their behavior. +- [ ] Frozen artifacts and shared fixtures guard the published v0.1 contract. +- [ ] Follow-up scope is reviewed after this slice is working, not assumed upfront. diff --git a/pyproject.toml b/pyproject.toml index 07664b095..fe9d45647 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "rich", "threadpoolctl; platform_machine != 'wasm32'", ] -version = "4.14.2.dev0" +version = "4.15.0.dev0" [project.entry-points."array_api"] blosc2 = "blosc2" diff --git a/pytest.ini b/pytest.ini index f566db577..d0a61ad55 100644 --- a/pytest.ini +++ b/pytest.ini @@ -38,3 +38,7 @@ markers = filterwarnings = error ignore::UserWarning + # Free-threaded CPython safely enables the GIL for this extension. This is + # an expected import notification, not a claim that no-GIL execution is safe. + # Keep the regex here: pytest's command-line -W treats the message literally. + ignore:The global interpreter lock \(GIL\) has been enabled to load module 'blosc2\.blosc2_ext':RuntimeWarning diff --git a/scripts/certify_menudet_math.py b/scripts/certify_menudet_math.py new file mode 100644 index 000000000..64c70db6d --- /dev/null +++ b/scripts/certify_menudet_math.py @@ -0,0 +1,174 @@ +"""Bounded, independent Menudet math certification (not a universal libm bound). + +Normal checks need only installed blosc2 and NumPy. Regeneration additionally +needs mpmath; references must agree at 100 and 200 decimal digits. Checked-in +vectors store exact input/output encodings, with per-function ULP budgets. +Run from any directory: python scripts/certify_menudet_math.py +""" + +import argparse +import json +import platform +from pathlib import Path + +import numpy as np + +DATA = Path(__file__).resolve().with_name("menudet_math_vectors.json") + + +def reference_cases(mp): + """Finite representative domains, including endpoints and cancellation.""" + unary = { + "acos": (-1, -0.75, 0.125, 0.875, 1), + "acosh": (1, 1.0009765625, 1.5, 8), + "asin": (-1, -0.75, 0.125, 0.875, 1), + "asinh": (-8, -0.125, 0.0009765625, 2), + "atan": (-8, -0.125, 0.0009765625, 2), + "atanh": (-0.875, -0.125, 0.0009765625, 0.875), + "cbrt": (-8, -0.125, 0.25, 2), + "cos": (-8, -0.125, 0.0009765625, 2), + "cosh": (-8, -0.125, 0.0009765625, 2), + "cospi": (-2, -0.75, 0.125, 0.5), + "erf": (-4, -0.125, 0.0009765625, 2), + "erfc": (-4, -0.125, 0.0009765625, 2), + "exp": (-8, -0.125, 0.0009765625, 2), + "exp10": (-8, -0.125, 0.0009765625, 2), + "exp2": (-8, -0.125, 0.0009765625, 2), + "expm1": (-8, -0.125, 0.0009765625, 2), + "lgamma": (0.125, 0.75, 1.5, 8), + "log": (0.0009765625, 0.75, 1.0009765625, 8), + "log10": (0.0009765625, 0.75, 1.0009765625, 8), + "log1p": (-0.875, -0.125, 0.0009765625, 8), + "log2": (0.0009765625, 0.75, 1.0009765625, 8), + "sin": (-8, -0.125, 0.0009765625, 2), + "sinh": (-8, -0.125, 0.0009765625, 2), + "sinpi": (-2, -0.75, 0.125, 0.5), + "sqrt": (0, 0.125, 1.0009765625, 8), + "tan": (-8, -0.125, 0.0009765625, 2), + "tanh": (-8, -0.125, 0.0009765625, 2), + "tgamma": (0.125, 0.75, 1.5, 8), + } + special = { + "cbrt": lambda x: mp.sign(x) * mp.root(abs(x), 3), + "cospi": mp.cospi, + "exp10": lambda x: mp.power(10, x), + "exp2": lambda x: mp.power(2, x), + "lgamma": lambda x: mp.log(abs(mp.gamma(x))), + "log10": lambda x: mp.log(x, 10), + "log2": lambda x: mp.log(x, 2), + "sinpi": mp.sinpi, + "tgamma": mp.gamma, + } + budgets = {"cospi": 8, "sinpi": 8, "exp10": 8, "lgamma": 8, "tgamma": 8} + for name, inputs in unary.items(): + function = special[name] if name in special else getattr(mp, name) + yield name, function, [(x,) for x in inputs], budgets.get(name, 4) + binary = { + "atan2": (mp.atan2, [(-2, -3), (0.125, -8), (2, 3)], 4), + "hypot": (lambda x, y: mp.sqrt(x * x + y * y), [(3, 4), (0.125, 8), (2, 3)], 4), + "logaddexp": (lambda x, y: mp.log(mp.exp(x) + mp.exp(y)), [(-8, -8), (-8, 8), (0.125, 0.25)], 8), + "pow": (mp.power, [(0.125, 1.5), (2, -0.75), (8, 0.125)], 4), + "fdim": (lambda x, y: max(x - y, mp.mpf(0)), [(8, 0.125), (0.125, 8)], 0), + "fmin": (min, [(-8, 0.125), (0.125, 8)], 0), + "fmax": (max, [(-8, 0.125), (0.125, 8)], 0), + "copysign": (lambda x, y: abs(x) * mp.sign(y), [(0.125, -8), (-8, 0.125)], 0), + "fmod": (lambda x, y: x - int(x / y) * y, [(-8.125, 2), (8.125, 2)], 0), + "remainder": (lambda x, y: x - mp.nint(x / y) * y, [(-7, 2), (7, 2)], 0), + "ldexp": (lambda x, y: x * mp.power(2, int(y)), [(0.125, -8), (1.5, 8)], 0), + "fma": (lambda x, y, z: x * y + z, [(1.0009765625, 0.9990234375, -1), (0.125, 8, -0.75)], 0), + } + for name, (function, inputs, budget) in binary.items(): + yield name, function, inputs, budget + + +def generate(): + import mpmath as mp + + cases = [] + for name, function, inputs, budget in reference_cases(mp): + for dtype in ("float32", "float64"): + for args in inputs: + typed = [float(np.asarray(x, dtype=dtype)) for x in args] + results = [] + for precision in (100, 200): + with mp.workdps(precision): + result = function(*(mp.mpf(x) for x in typed)) + results.append(float(np.asarray(float(result), dtype=dtype)).hex()) + if results[0] != results[1]: + raise ValueError(f"Unstable reference: {name} {dtype} {typed}") + cases.append( + { + "function": name, + "dtype": dtype, + "inputs": [x.hex() for x in typed], + "expected": results[0], + "max_ulp": budget, + } + ) + return { + "generator": f"mpmath {mp.__version__}, 100/200 decimal digits", + "scope": "Finite inputs only; not a global accuracy or bitwise portability guarantee", + "cases": cases, + } + + +def check(record): + import blosc2 + + failures = [] + maxima = {} + for case in record["cases"]: + name, dtype = case["function"], case["dtype"] + names = list("xyz"[: len(case["inputs"])]) + signature = dict.fromkeys(names, dtype) + if name == "ldexp": + signature["y"] = "int64" + source = f"def k({', '.join(names)}):\n return {name}({', '.join(names)})\n" + artifact = blosc2.DSLKernel.from_source(source).export(signature, dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + inputs = { + key: np.array([float.fromhex(value)], dtype=signature[key]) + for key, value in zip(names, case["inputs"], strict=True) + } + actual = float(kernel.evaluate(inputs)[0]) + expected = float.fromhex(case["expected"]) + # Use the smaller adjacent-format gap, including at powers of two. Python + # doubles represent all tested f32/f64 gaps/differences exactly enough here. + ref = np.asarray(expected, dtype=dtype) + upper = float(np.nextafter(ref, np.asarray(np.inf, dtype=dtype))) + lower = float(np.nextafter(ref, np.asarray(-np.inf, dtype=dtype))) + gap = min(upper - expected, expected - lower) + error = abs(actual - expected) / gap + key = f"{name}/{dtype}" + maxima[key] = max(maxima.get(key, 0), error) + if not np.isfinite(actual) or error > case["max_ulp"]: + failures.append({**case, "actual": actual.hex(), "error_ulp": error}) + return { + "package": blosc2.__file__, + "version": blosc2.__version__, + "platform": platform.platform(), + "reference": record["generator"], + "samples": len(record["cases"]), + "max_observed_ulp": maxima, + "failures": failures, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--generate", action="store_true", help="regenerate independent reference vectors") + parser.add_argument("--report", type=Path, help="write machine-readable certification report") + args = parser.parse_args() + if args.generate: + DATA.write_text(json.dumps(generate(), indent=2) + "\n") + return + report = check(json.loads(DATA.read_text())) + if args.report: + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + if report["failures"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/menudet_math_vectors.json b/scripts/menudet_math_vectors.json new file mode 100644 index 000000000..9b3a3abef --- /dev/null +++ b/scripts/menudet_math_vectors.json @@ -0,0 +1,2622 @@ +{ + "generator": "mpmath 1.3.0, 100/200 decimal digits", + "scope": "Finite inputs only; not a global accuracy or bitwise portability guarantee", + "cases": [ + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "0x1.921fb60000000p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "0x1.359d260000000p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.720a3a0000000p+0", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.02be9c0000000p-1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "0x1.921fb54442d18p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "0x1.359d26f93b6c3p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.720a392c1d955p+0", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.02be9ce0b87cdp-1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.6a025c0000000p-5", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.ecc2ca0000000p-1", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.62636e0000000p+1", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.6a025bf510baep-5", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.ecc2caec5160ap-1", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.62636e408c33bp+1", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.921fb60000000p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.b235320000000p-1", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.00abe00000000p-3", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.10c0660000000p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x1.921fb60000000p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.921fb54442d18p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.b235315c680dcp-1", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.00abe0c129e1ep-3", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.10c066d3e6932p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x1.921fb54442d18p+0", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.6363720000000p+1", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fead0c0000000p-4", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffa0000000p-11", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.7192180000000p+0", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.63637195fabd3p+1", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fead0b6996973p-4", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffaaaaad11p-11", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.719218313d087p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.7249fa0000000p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd5baa0000000p-4", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff60000000p-11", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.1b6e1a0000000p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.7249faa996a21p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd5ba9aac2f6ep-4", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff55555bbcp-11", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.1b6e192ebbe44p+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.5aa1640000000p+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.0158920000000p-3", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000060000000p-10", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.5aa1640000000p+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.5aa16394d481fp+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.015891c9eaef7p-3", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000055555889p-10", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.5aa16394d481fp+0", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p+1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-2" + ], + "expected": "0x1.428a300000000p-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.428a300000000p+0", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p+1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-2" + ], + "expected": "0x1.428a2f98d728bp-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.428a2f98d728bp+0", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.29fbec0000000p-3", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.fc01560000000p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff00000000p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.aa22660000000p-2", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.29fbebf632f94p-3", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.fc015527d5bd3p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff00000155p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.aa22657537205p-2", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.749eaa0000000p+10", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.0200aa0000000p+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000080000000p+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.e18fa00000000p+1", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.749eaa93f4e76p+10", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.0200aac16db6fp+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.00000800000abp+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.e18fa0df2d9bcp+1", + "max_ulp": 4 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e60000000p-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.d906bc0000000p-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e667f3bcdp-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.d906bcf328d46p-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "-0x1.0000000000000p+0", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1f5e1a0000000p-3", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.20dd6e0000000p-10", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.fd9ae20000000p-1", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "-0x1.ffffff7b91176p-1", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1f5e1a35c3b89p-3", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.20dd6eff8cb8fp-10", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.fd9ae142795e3p-1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "0x1.0000000000000p+1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.23ebc40000000p+0", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ff6f920000000p-1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.328f5e0000000p-8", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "0x1.ffffffbdc88bbp+0", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.23ebc346b8771p+0", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ff6f91488039ap-1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.328f5ec350e67p-8", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5fc2100000000p-12", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.c3d6a20000000p-1", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0040080000000p+0", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d8e64c0000000p+2", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5fc21041027adp-12", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.c3d6a24ed8222p-1", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.00400800aab55p+0", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d8e64b8d4ddaep+2", + "max_ulp": 4 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5798ee0000000p-27", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.7ff2220000000p-1", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0093880000000p+0", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.9000000000000p+6", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5798ee2308c3ap-27", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.7ff2224115d9ap-1", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.009388004be7ep+0", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.9000000000000p+6", + "max_ulp": 8 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-8", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.d5818e0000000p-1", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.002c600000000p+0", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+2", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-8", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.d5818dcfba487p-1", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.002c605e2e8cfp+0", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+2", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.ffd4080000000p-1", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.e14aee0000000p-4", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0020020000000p-10", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.98e64c0000000p+2", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.ffd407bdf7dfbp-1", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.e14aed893eef4p-4", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.002002aad5577p-10", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.98e64b8d4ddaep+2", + "max_ulp": 4 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.027c4c0000000p+1", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.a051c40000000p-3", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "-0x1.eeb95c0000000p-4", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.10ce200000000p+3", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.027c4cfd515b0p+1", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.a051c372609eep-3", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "-0x1.eeb95b094c191p-4", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.10ce1f32dcc30p+3", + "max_ulp": 8 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.bb9d3c0000000p+2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.2696220000000p-2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.ffc00a0000000p-11", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.0a2b240000000p+1", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.bb9d3beb8c86bp+2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.269621134db92p-2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.ffc00aa8ab110p-11", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.0a2b23f3bab73p+1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.8151820000000p+1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.ffbfc20000000p-4", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.bc80240000000p-12", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.ce61d00000000p-1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.8151824c7587fp+1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.ffbfc2bbc7803p-4", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.bc80239e5b560p-12", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.ce61cf8ef36fep-1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.0a2b240000000p+1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1178e80000000p-3", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ffc00a0000000p-11", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.193ea80000000p+1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.0a2b23f3bab73p+1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1178e8227e47cp-3", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ffc00aa8ab110p-11", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.193ea7aad030bp+1", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.4000000000000p+3", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.a8ff980000000p-2", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.7126540000000p-10", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+1", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.4000000000000p+3", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.a8ff971810a5ep-2", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.712653743f454p-10", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fa8d2a0000000p-1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.feaaee0000000p-4", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffa0000000p-11", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d18f6e0000000p-1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fa8d2a028cf7bp-1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.feaaeee86ee36p-4", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffaaaaaaefp-11", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d18f6ead1b446p-1", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.749ea60000000p+10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.00aacc0000000p-3", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000020000000p-10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d03cf60000000p+1", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.749ea514eca66p+10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.00aaccd00d2f1p-3", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.000002aaaaacdp-10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d03cf63b6e19fp+1", + "max_ulp": 4 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e60000000p-1", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.87de2a0000000p-2", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e667f3bcdp-1", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.87de2a6aea963p-2", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x0.0p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.6a09e60000000p-2", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.001ffe0000000p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.6a09e60000000p+1", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x0.0p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.6a09e667f3bcdp-2", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.001ffe003ff60p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.6a09e667f3bcdp+1", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.b32e780000000p+2", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.01577a0000000p-3", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000060000000p-10", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.17af620000000p+1", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.b32e78f49a1e4p+2", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.01577af1511a5p-3", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000055555777p-10", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.17af62e0950f8p+1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fffff80000000p-1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd59920000000p-4", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff60000000p-11", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.ed95060000000p-1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fffff872a91f8p-1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd5992bc4b835p-4", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff5555599ap-11", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.ed9505e1bc3d4p-1", + "max_ulp": 4 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.e22c1a0000000p+2", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.39b4e80000000p+0", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.c5bf8a0000000p-1", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.3b00000000000p+12", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.e22c196233d21p+2", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.39b4e8b50f62cp+0", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.c5bf891b4ef6bp-1", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.3b00000000000p+12", + "max_ulp": 8 + }, + { + "function": "atan2", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+1", + "-0x1.8000000000000p+1" + ], + "expected": "-0x1.46dc0a0000000p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.901fc00000000p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.2d0eae0000000p-1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+1", + "-0x1.8000000000000p+1" + ], + "expected": "-0x1.46dc09ec29433p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.901fbfee871a6p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.2d0ead6066395p-1", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+1", + "0x1.0000000000000p+2" + ], + "expected": "0x1.4000000000000p+2", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0008000000000p+3", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.cd82b40000000p+1", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+1", + "0x1.0000000000000p+2" + ], + "expected": "0x1.4000000000000p+2", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0007ffe000fffp+3", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.cd82b446159f3p+1", + "max_ulp": 4 + }, + { + "function": "logaddexp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.d3a37a0000000p+2", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p-2" + ], + "expected": "0x1.c3e4060000000p-1", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.d3a37a020b8c2p+2", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000003c6ab74p+3", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p-2" + ], + "expected": "0x1.c3e4055056374p-1", + "max_ulp": 8 + }, + { + "function": "pow", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.8000000000000p+0" + ], + "expected": "0x1.6a09e60000000p-5", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.306fe00000000p-1", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.4bfdae0000000p+0", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.8000000000000p+0" + ], + "expected": "0x1.6a09e667f3bcdp-5", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.306fe0a31b715p-1", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.4bfdad5362a27p+0", + "max_ulp": 4 + }, + { + "function": "fdim", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.f800000000000p+2", + "max_ulp": 0 + }, + { + "function": "fdim", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x0.0p+0", + "max_ulp": 0 + }, + { + "function": "fdim", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.f800000000000p+2", + "max_ulp": 0 + }, + { + "function": "fdim", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x0.0p+0", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float32", + "inputs": [ + "-0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float32", + "inputs": [ + "0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float64", + "inputs": [ + "-0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float64", + "inputs": [ + "0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float32", + "inputs": [ + "-0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float64", + "inputs": [ + "-0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-11", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0", + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+8", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-11", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0", + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+8", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0", + "0x1.ff80000000000p-1", + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.0000000000000p-20", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.0000000000000p-2", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0", + "0x1.ff80000000000p-1", + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.0000000000000p-20", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.0000000000000p-2", + "max_ulp": 0 + } + ] +} diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 39008fd0b..3fa503c49 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -586,7 +586,15 @@ def _raise(exc): from .c2array import c2context, C2Array, C2NDSource, ChunkAlreadyWritten, URLPath -from .dsl_kernel import DSLSyntaxError, DSLKernel, dsl_kernel, validate_dsl, validate_dsl_jit +from .dsl_kernel import ( + DSLSyntaxError, + DSLKernel, + dsl_kernel, + validate_dsl, + validate_dsl_jit, + validate_portable_dsl, +) +from .portable_kernel import PortableArtifactError, PortableKernel from .jit_config import get_jit_options, jit_options, set_jit_options from .lazyexpr import ( LazyExpr, @@ -908,6 +916,9 @@ def _raise(exc): "Index", "LazyArray", "DSLKernel", + "validate_portable_dsl", + "PortableKernel", + "PortableArtifactError", "DSLSyntaxError", "LazyExpr", "LazyUDF", diff --git a/src/blosc2/b2objects.py b/src/blosc2/b2objects.py index fc4c2cac7..d49a617ff 100644 --- a/src/blosc2/b2objects.py +++ b/src/blosc2/b2objects.py @@ -7,16 +7,15 @@ from __future__ import annotations -import inspect import pathlib -import textwrap -from dataclasses import asdict from typing import Any import numpy as np import blosc2 -from blosc2.dsl_kernel import DSLKernel, kernel_from_source +from blosc2.deserialization import DeserializeMode, get_deserialize, normalize_deserialize +from blosc2.dsl_kernel import kernel_from_source +from blosc2.exceptions import UnsafeDeserializationError _B2OBJECT_META_KEY = "b2o" _B2OBJECT_VERSION = 1 @@ -24,6 +23,23 @@ _B2OBJECT_USER_VLMETA_KEY = "_b2o_user_vlmeta" +def preflight_persistence(obj): + """Validate executable descriptors recursively before a destination is touched.""" + if isinstance(obj, blosc2.LazyArray): + encode_b2object_payload(obj) + from blosc2.msgpack_utils import msgpack_packb + + msgpack_packb(obj._get_user_vlmeta()) + elif isinstance(obj, blosc2.CTable): + obj._preflight_portable_persistence() + else: + schunk = getattr(obj, "schunk", obj if isinstance(obj, blosc2.SChunk) else None) + if schunk is not None: + marker = schunk.meta.get("b2o", {}) + if marker.get("kind") == "lazyudf" or schunk.meta.get("LazyArray") == 1: + raise TypeError("Persisting legacy DSL/Python UDF metadata is unsupported") + + def make_b2object_carrier( kind: str, shape, @@ -55,10 +71,14 @@ def read_b2object_user_vlmeta(obj) -> dict[str, Any]: def encode_operand_reference(obj): + from blosc2.portable_lazy import PortableLazyArray + + if isinstance(obj, PortableLazyArray | blosc2.LazyExpr | blosc2.LazyUDF): + return encode_b2object_payload(obj) return blosc2.Ref.from_object(obj).to_dict() -def decode_operand_reference(payload, *, base_path=None): +def decode_operand_reference(payload, *, base_path=None, deserialize="safe"): if ( payload.get("kind") in {"urlpath", "dictstore_key"} and base_path is not None @@ -67,10 +87,21 @@ def decode_operand_reference(payload, *, base_path=None): payload = dict(payload) payload["urlpath"] = (base_path / payload["urlpath"]).as_posix() ref = blosc2.Ref.from_dict(payload) - return ref.open(deserialize="full") + return ref.open(deserialize=deserialize) def encode_b2object_payload(obj) -> dict[str, Any] | None: + from blosc2.portable_lazy import PortableLazyArray + + if isinstance(obj, PortableLazyArray): + return { + "kind": "portable", + "version": _B2OBJECT_VERSION, + "artifact": obj.kernel.to_json(), + "shape": list(obj._domain), + "partitions": list(obj.partitions), + "operands": {name: encode_operand_reference(value) for name, value in obj.inputs.items()}, + } if isinstance(obj, blosc2.C2Array): return blosc2.Ref.c2array_ref(obj.path, obj.urlbase).to_dict() if isinstance(obj, blosc2.LazyExpr): @@ -83,66 +114,55 @@ def encode_b2object_payload(obj) -> dict[str, Any] | None: "operands": {key: encode_operand_reference(value) for key, value in operands.items()}, } if isinstance(obj, blosc2.LazyUDF): - if not isinstance(obj.func, DSLKernel): - raise TypeError("Structured Blosc2 msgpack payload only supports LazyUDF backed by DSLKernel") - udf_func = obj.func.func - udf_name = getattr(udf_func, "__name__", obj.func.__name__) - try: - udf_source = textwrap.dedent(inspect.getsource(udf_func)).lstrip() - except Exception: - udf_source = obj.func.dsl_source - if udf_source is None: - raise ValueError("Structured LazyUDF msgpack payload requires recoverable DSL kernel source") - kwargs = {} - for key, value in obj.kwargs.items(): - if key in {"dtype", "shape"}: - continue - if isinstance(value, blosc2.CParams | blosc2.DParams): - kwargs[key] = asdict(value) - else: - kwargs[key] = value - return { - "kind": "lazyudf", - "version": _B2OBJECT_VERSION, - "function_kind": "dsl", - "dsl_version": _B2OBJECT_DSL_VERSION, - "name": udf_name, - "udf_source": udf_source, - "dtype": np.dtype(obj.dtype).str, - "shape": list(obj.shape), - "operands": {f"o{i}": encode_operand_reference(value) for i, value in enumerate(obj.inputs)}, - "kwargs": kwargs, - } + from blosc2.portable_lazy import portable_from_lazyudf + + return encode_b2object_payload(portable_from_lazyudf(obj)) return None -def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carrier=None): +def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carrier=None, deserialize="safe"): + deserialize = normalize_deserialize(deserialize) kind = payload.get("kind") version = payload.get("version") if version != _B2OBJECT_VERSION: raise ValueError(f"Unsupported persisted Blosc2 object version: {version!r}") if kind == "c2array": ref = blosc2.Ref.from_dict(payload) - return ref.open(deserialize="full") + return ref.open(deserialize=deserialize) + if kind == "portable": + kernel = blosc2.PortableKernel.from_json(payload["artifact"]) + operands, missing = decode_operand_mapping( + payload["operands"], base_path=carrier_path, deserialize=deserialize + ) + if missing: + raise FileNotFoundError(f"Missing portable operands: {missing}") + return kernel.lazy(operands, shape=payload["shape"], partitions=payload["partitions"]) if kind == "remote_array": if carrier is None: raise ValueError("A persisted RemoteArray requires its B2ND carrier") return blosc2.RemoteArray._from_payload(payload, carrier) if kind == "lazyexpr": - return decode_structured_lazyexpr(payload, carrier_path=carrier_path) + return decode_structured_lazyexpr(payload, carrier_path=carrier_path, deserialize=deserialize) if kind == "lazyudf": - return decode_structured_lazyudf(payload, carrier_path=carrier_path) + if deserialize is not DeserializeMode.FULL: + raise UnsafeDeserializationError("legacy DSL LazyUDF") + return decode_structured_lazyudf(payload, carrier_path=carrier_path, deserialize=deserialize) raise ValueError(f"Unsupported persisted Blosc2 object kind: {kind!r}") -def decode_structured_lazyexpr(payload, *, carrier_path=None): +def decode_structured_lazyexpr(payload, *, carrier_path=None, deserialize="safe"): expression = payload.get("expression") if not isinstance(expression, str): raise TypeError("Structured LazyExpr payload requires a string 'expression'") operands_payload = payload.get("operands") if not isinstance(operands_payload, dict): raise TypeError("Structured LazyExpr payload requires a mapping 'operands'") - operands, missing_ops = decode_operand_mapping(operands_payload, base_path=carrier_path) + from blosc2.lazyexpr import validate_expr + + validate_expr(expression) + operands, missing_ops = decode_operand_mapping( + operands_payload, base_path=carrier_path, deserialize=deserialize + ) if missing_ops: exc = blosc2.exceptions.MissingOperands(expression, missing_ops) exc.expr = expression @@ -151,12 +171,17 @@ def decode_structured_lazyexpr(payload, *, carrier_path=None): return blosc2.lazyexpr(expression, operands=operands) -def decode_operand_mapping(operands_payload, *, base_path=None): +def decode_operand_mapping(operands_payload, *, base_path=None, deserialize="safe"): operands = {} missing_ops = {} for key, value in operands_payload.items(): try: - operands[key] = decode_operand_reference(value, base_path=base_path) + if value.get("kind") in {"portable", "lazyexpr", "lazyudf"}: + operands[key] = decode_b2object_payload( + value, carrier_path=base_path, deserialize=deserialize + ) + else: + operands[key] = decode_operand_reference(value, base_path=base_path, deserialize=deserialize) except FileNotFoundError: ref = blosc2.Ref.from_dict(value) if ref.kind in {"urlpath", "dictstore_key"}: @@ -166,7 +191,9 @@ def decode_operand_mapping(operands_payload, *, base_path=None): return operands, missing_ops -def decode_structured_lazyudf(payload, *, carrier_path=None): +def decode_structured_lazyudf(payload, *, carrier_path=None, deserialize="safe"): + if normalize_deserialize(deserialize) is not DeserializeMode.FULL: + raise UnsafeDeserializationError("legacy DSL LazyUDF") function_kind = payload.get("function_kind") if function_kind != "dsl": raise ValueError(f"Unsupported structured LazyUDF function kind: {function_kind!r}") @@ -194,15 +221,19 @@ def decode_structured_lazyudf(payload, *, carrier_path=None): func = kernel_from_source(udf_source, name) ordered_operands_payload = {f"o{n}": operands_payload[f"o{n}"] for n in range(len(operands_payload))} - operands, missing_ops = decode_operand_mapping(ordered_operands_payload, base_path=carrier_path) + operands, missing_ops = decode_operand_mapping( + ordered_operands_payload, base_path=carrier_path, deserialize=deserialize + ) if missing_ops: exc = blosc2.exceptions.MissingOperands(name, missing_ops) exc.expr = name exc.missing_ops = missing_ops raise exc - return blosc2.lazyudf( + result = blosc2.lazyudf( func, tuple(operands.values()), dtype=np.dtype(dtype), shape=tuple(shape_payload), **kwargs ) + result._legacy_source_recipe = True + return result def read_b2object_marker(obj) -> dict[str, Any] | None: @@ -231,8 +262,13 @@ def open_b2object(obj): schunk = getattr(obj, "schunk", obj) if getattr(schunk, "urlpath", None) is not None: carrier_path = pathlib.Path(schunk.urlpath).parent - opened = decode_b2object_payload(payload, carrier_path=carrier_path, carrier=obj) - if isinstance(opened, blosc2.LazyExpr | blosc2.LazyUDF): + opened = decode_b2object_payload( + payload, carrier_path=carrier_path, carrier=obj, deserialize=get_deserialize(obj) + ) + from blosc2.deserialization import set_deserialize + + set_deserialize(opened, get_deserialize(obj)) + if isinstance(opened, blosc2.LazyArray): opened.array = obj opened.schunk = schunk opened._set_user_vlmeta(read_b2object_user_vlmeta(obj), sync=False) diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index 64affa623..fd17e3245 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -961,6 +961,340 @@ cdef inline object _numpy_dtype_from_me_dtype(me_dtype dt): return None +cdef extern from *: + """ + #include "miniexpr.h" + #include + #include + static int b2_validate_portable_dsl(const char *source, const char *version, + const me_variable *inputs, int ninputs, me_dtype output_dtype, + size_t output_itemsize, int ndim, int cardinality, + int *line, int *column, char *message, size_t capacity) { + #ifdef ME_PORTABLE_DSL_VALIDATION_DESCRIPTOR_VERSION + if (strcmp(ME_PORTABLE_DSL_VERSION, "1.0")) { + *line = *column = 0; + snprintf(message, capacity, "installed miniexpr lacks draft 1.0 source validation"); + return -100; + } + me_portable_error error = {0}; + int rc; + me_portable_validation_descriptor descriptor = { + .struct_size = sizeof(descriptor), + .version = ME_PORTABLE_DSL_VALIDATION_DESCRIPTOR_VERSION, + .output_itemsize = output_itemsize, .ndim = ndim, + .cardinality = (me_portable_cardinality)cardinality}; + rc = me_validate_portable_dsl_ex(source, version, inputs, ninputs, + output_dtype, &descriptor, &error); + *line = error.line; + *column = error.column; + snprintf(message, capacity, "%s", error.message); + return rc; + #else + *line = *column = 0; + snprintf(message, capacity, "miniexpr was built without draft 1.0 source validation support"); + return -100; + #endif + } + """ + int b2_validate_portable_dsl(const char *source, const char *version, + const me_variable *inputs, int ninputs, me_dtype output_dtype, + size_t output_itemsize, int ndim, int cardinality, + int *line, int *column, char *message, size_t capacity) + + +def validate_portable_dsl_source(source, input_dtypes, output_dtype, version, int ndim=0, int cardinality=0): + """Native-only versioned source validation; no arrays, JIT or Python execution.""" + cdef Py_ssize_t n = len(input_dtypes) + if n > 128: + raise ValueError("Too many portable DSL inputs (maximum 128)") + cdef me_variable *variables = NULL + cdef list names = [name.encode("utf-8") for name in input_dtypes] + cdef bytes source_bytes = source.encode("utf-8") + cdef bytes version_bytes = version.encode("utf-8") + cdef bytes name_bytes + cdef np.dtype dtype + cdef me_dtype result_dtype + cdef size_t output_itemsize = 0 + cdef int rc, line = 0, column = 0 + cdef char message[256] + cdef Py_ssize_t i + if n: + variables = calloc(n, sizeof(me_variable)) + if variables == NULL: + raise MemoryError() + try: + for i, value in enumerate(input_dtypes.values()): + name_bytes = names[i] + variables[i].name = name_bytes + dtype = np.dtype(value) + try: + variables[i].dtype = _me_dtype_from_numpy_dtype(dtype) + except TypeError: + variables[i].dtype = ME_AUTO + if version == "1.0" and variables[i].dtype in (ME_STRING, ME_BYTES): + variables[i].itemsize = dtype.itemsize + dtype = np.dtype(output_dtype) + try: + result_dtype = _me_dtype_from_numpy_dtype(dtype) + except TypeError: + result_dtype = ME_AUTO + if version == "1.0" and result_dtype in (ME_STRING, ME_BYTES): + output_itemsize = dtype.itemsize + rc = b2_validate_portable_dsl(source_bytes, version_bytes, variables, n, + result_dtype, output_itemsize, ndim, cardinality, + &line, &column, message, sizeof(message)) + statuses = { + 0: "success", -1: "unsupported_version", -2: "invalid_signature", + -3: "invalid_source", -4: "unsupported_feature", -5: "out_of_memory", + -100: "runtime_unsupported", + } + return {"valid": rc == 0, "status": statuses.get(rc, "native_error"), + "line": line, "column": column, + "error": None if rc == 0 else (message).decode("utf-8", "replace")} + finally: + free(variables) + + +cdef extern from "dsl_artifact_bridge.h": + ctypedef struct b2_artifact_input: + const char *name + me_dtype dtype + const void *data + size_t nitems + ctypedef struct b2_artifact_error: + int native_status + int line + int column + char message[256] + int b2_artifact_available() + int b2_artifact_load(const char *, size_t, int, void **, b2_artifact_error *) nogil + int b2_artifact_eval(const void *, const b2_artifact_input *, int, void *, size_t, + b2_artifact_error *) nogil + void b2_artifact_free(void *) noexcept nogil + const char *b2_artifact_source(const void *) + const char *b2_artifact_entry(const void *) + int b2_artifact_ninputs(const void *) + const char *b2_artifact_name(const void *, int) + me_dtype b2_artifact_dtype(const void *, int) + me_dtype b2_artifact_output(const void *) + int b2_artifact_jit(const void *) + ctypedef struct b2_artifact_buffer: + const char *name + me_dtype dtype + size_t itemsize + const void *data + size_t capacity + ctypedef struct b2_artifact_descriptor: + size_t struct_size + unsigned int version + size_t nitems + size_t output_capacity + const uint8_t *valid_mask + size_t valid_mask_capacity + int ndim + const int64_t *logical_shape + const int64_t *block_origin + const int64_t *block_extent + int b2_artifact_descriptor_available() + int b2_artifact_cardinality(const void *) + size_t b2_artifact_width(const void *, int) + int b2_artifact_rank(const void *) + const char *b2_artifact_version(const void *) + int b2_artifact_eval_ex(const void *, const b2_artifact_buffer *, int, void *, + const b2_artifact_descriptor *, b2_artifact_error *) nogil + + +def portable_artifact_available(): + return bool(b2_artifact_available()) + + +def portable_descriptor_available(): + return bool(b2_artifact_descriptor_available()) + + +cdef void raise_artifact_error(int rc, b2_artifact_error *error) except *: + from blosc2.portable_kernel import PortableArtifactError + + if rc == -100: + raise NotImplementedError("Build with miniexpr portable artifact support") + if rc == -6: + raise MemoryError((error.message).decode("utf-8", "replace")) + statuses = {-1: "invalid_artifact", -2: "unsupported_requirement", -3: "invalid_source", + -4: "binding_error", -5: "evaluation_error"} + raise PortableArtifactError( + (error.message).decode("utf-8", "replace"), + status=statuses.get(rc, "native_error"), native_status=error.native_status, + line=error.line, column=error.column) + + +cdef class PortableArtifactHandle: + """Owned immutable native handle; no Python artifact decoding or source execution.""" + cdef void *_handle + + def __cinit__(self, bytes artifact, int jit_mode): + cdef b2_artifact_error error + cdef int rc + cdef const char *json = artifact + cdef size_t size = len(artifact) + self._handle = NULL + with nogil: + rc = b2_artifact_load(json, size, jit_mode, &self._handle, &error) + if rc: + raise_artifact_error(rc, &error) + + def __dealloc__(self): + b2_artifact_free(self._handle) + + def info(self): + cdef int i + return { + "source": (b2_artifact_source(self._handle)).decode("utf-8"), + "entry_point": (b2_artifact_entry(self._handle)).decode("utf-8"), + "inputs": {(b2_artifact_name(self._handle, i)).decode("utf-8"): + _numpy_dtype_from_me_dtype(b2_artifact_dtype(self._handle, i)) + for i in range(b2_artifact_ninputs(self._handle))}, + "output_dtype": _numpy_dtype_from_me_dtype(b2_artifact_output(self._handle)), + "jit": bool(b2_artifact_jit(self._handle)), + } + + cdef object signature_dtype(self, int index): + cdef me_dtype dtype = b2_artifact_output(self._handle) if index < 0 else b2_artifact_dtype(self._handle, index) + cdef size_t width = b2_artifact_width(self._handle, index) + if dtype == ME_STRING: + return np.dtype(f"U{width // 4}") + if dtype == ME_BYTES: + return np.dtype(f"S{width}") + return _numpy_dtype_from_me_dtype(dtype) + + def descriptor_info(self): + """Authoritative draft native signature; unavailable on older pinned builds.""" + if not b2_artifact_descriptor_available(): + raise NotImplementedError("Build with draft miniexpr descriptor support") + return { + "schema_version": (b2_artifact_version(self._handle)).decode("ascii"), + "inputs": {(b2_artifact_name(self._handle, i)).decode("utf-8"): + self.signature_dtype(i) for i in range(b2_artifact_ninputs(self._handle))}, + "output_dtype": self.signature_dtype(-1), + "cardinality": "block_scalar" if b2_artifact_cardinality(self._handle) == 1 else "elementwise", + "ndim": b2_artifact_rank(self._handle), + } + + def evaluate_block(self, inputs, block_shape, *, logical_shape=None, block_origin=None, valid_mask=None): + """Evaluate one explicit logical block; never infer ND coordinates from a physical tile.""" + from math import prod + + if not b2_artifact_descriptor_available(): + raise NotImplementedError("Build with draft miniexpr descriptor support") + cdef Py_ssize_t n = len(inputs), i + if n > 128: + raise ValueError("Too many portable artifact inputs") + block_shape = tuple(block_shape) + if any(type(size) is not int or size < 0 for size in block_shape): + raise ValueError("Block shape must contain nonnegative Python integers") + count = prod(block_shape) + if count > 2147483647: + raise ValueError("Native block exceeds the supported lane count") + cdef b2_artifact_buffer *bindings = NULL + cdef b2_artifact_descriptor descriptor + cdef b2_artifact_error error + cdef np.ndarray array, output, mask, shape_array, origin_array, extent_array + cdef list names = [key.encode("utf-8") for key in inputs] + cdef list arrays = list(inputs.values()) + cdef bytes name + cdef int rc + memset(&descriptor, 0, sizeof(descriptor)) + descriptor.struct_size = sizeof(descriptor) + descriptor.version = 1 + descriptor.nitems = count + descriptor.ndim = b2_artifact_rank(self._handle) + if descriptor.ndim: + if logical_shape is None or block_origin is None or len(block_shape) != descriptor.ndim: + raise ValueError("Explicit logical shape, block origin and matching block rank are required") + shape_array = np.ascontiguousarray(logical_shape, dtype=np.int64) + origin_array = np.ascontiguousarray(block_origin, dtype=np.int64) + extent_array = np.ascontiguousarray(block_shape, dtype=np.int64) + if shape_array.ndim != 1 or origin_array.ndim != 1 or shape_array.shape[0] != descriptor.ndim or origin_array.shape[0] != descriptor.ndim: + raise ValueError("Logical context rank mismatch") + descriptor.logical_shape = np.PyArray_DATA(shape_array) + descriptor.block_origin = np.PyArray_DATA(origin_array) + descriptor.block_extent = np.PyArray_DATA(extent_array) + elif logical_shape is not None or block_origin is not None: + raise ValueError("This artifact does not declare ND context") + if valid_mask is not None: + mask = np.asarray(valid_mask) + if mask.dtype not in (np.dtype(np.bool_), np.dtype(np.uint8)) or mask.size != count: + raise ValueError("Valid mask must have one Boolean/uint8 slot per block lane") + mask = np.ascontiguousarray(mask, dtype=np.uint8) + descriptor.valid_mask = np.PyArray_DATA(mask) + descriptor.valid_mask_capacity = mask.nbytes + output = np.empty(() if b2_artifact_cardinality(self._handle) == 1 else block_shape, + dtype=self.signature_dtype(-1)) + descriptor.output_capacity = output.nbytes + if n: + bindings = calloc(n, sizeof(b2_artifact_buffer)) + if bindings == NULL: + raise MemoryError() + try: + for i in range(n): + array = arrays[i] + if not array.flags.c_contiguous or not array.flags.aligned or not array.dtype.isnative or array.size != count: + raise ValueError("Block inputs must be aligned, contiguous, host-endian and match the block extent") + name = names[i] + if b'\x00' in name: + raise ValueError("Input names must not contain NUL") + bindings[i].name = name + bindings[i].dtype = _me_dtype_from_numpy_dtype(array.dtype) + bindings[i].itemsize = array.dtype.itemsize + bindings[i].data = np.PyArray_DATA(array) + bindings[i].capacity = array.nbytes + with nogil: + rc = b2_artifact_eval_ex(self._handle, bindings, n, np.PyArray_DATA(output), &descriptor, &error) + if rc: + raise_artifact_error(rc, &error) + return output + finally: + free(bindings) + + def evaluate(self, inputs, shape): + cdef Py_ssize_t n = len(inputs) + if n > 128: + raise ValueError("Too many portable artifact inputs") + cdef b2_artifact_input *bindings = NULL + cdef b2_artifact_error error + cdef np.ndarray array, output + cdef list names = [key.encode("utf-8") for key in inputs] + cdef list arrays = list(inputs.values()) + cdef bytes name + cdef int rc + cdef Py_ssize_t i + cdef size_t count + output = np.empty(shape, dtype=_numpy_dtype_from_me_dtype(b2_artifact_output(self._handle))) + count = output.size + if n: + bindings = calloc(n, sizeof(b2_artifact_input)) + if bindings == NULL: + raise MemoryError() + try: + for i in range(n): + array = arrays[i] + if not array.flags.c_contiguous or not array.flags.aligned or not array.dtype.isnative: + raise ValueError("Native artifact inputs must be aligned, contiguous, and host-endian") + name = names[i] + if b'\x00' in name: + raise ValueError("Input names must not contain NUL") + bindings[i].name = name + bindings[i].dtype = _me_dtype_from_numpy_dtype(array.dtype) + bindings[i].data = np.PyArray_DATA(array) + bindings[i].nitems = array.size + with nogil: + rc = b2_artifact_eval(self._handle, bindings, n, np.PyArray_DATA(output), count, &error) + if rc: + raise_artifact_error(rc, &error) + return output + finally: + free(bindings) + + def me_output_dtype(expression, operands): """Ask miniexpr what dtype *expression* would produce over *operands*. diff --git a/src/blosc2/core.py b/src/blosc2/core.py index b2ebfb75e..a3f250197 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -2392,7 +2392,7 @@ def from_cframe( if "b2o" in schunk.meta: marker = schunk.meta["b2o"] kind = marker.get("kind", "b2o") if isinstance(marker, dict) else "b2o" - if deserialize is DeserializeMode.SAFE: + if deserialize is DeserializeMode.SAFE and kind not in {"portable", "lazyexpr"}: raise UnsafeDeserializationError(str(kind)) return blosc2.open_b2object(ndarray_from_cframe(cframe, copy=copy, deserialize=deserialize)) if "b2nd" in schunk.meta: diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 64cdb8032..577c8d0df 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -7007,6 +7007,7 @@ def to_b2z( """ if not str(urlpath).endswith(".b2z"): raise ValueError("urlpath must have a .b2z extension") + self._preflight_portable_persistence() if preserve_sources and self.base is not None: raise ValueError("preserve_sources requires an unfiltered root table") @@ -7075,6 +7076,7 @@ def to_b2d( """ urlpath = os.fspath(urlpath) storage = getattr(self, "_storage", None) + self._preflight_portable_persistence() can_physical_unpack = ( not compact and self.base is None @@ -7122,6 +7124,7 @@ def to_cframe(self, *, preserve_sources: bool = False) -> bytes: :func:`blosc2.ctable_from_cframe` """ # Materialize live rows for views/slices; for base tables use the live + self._preflight_portable_persistence() # columns directly. copy() is the canonical materialization path. if self.base is not None: src = self.copy(compact=True) @@ -7421,6 +7424,7 @@ def save(self, urlpath: str, *, overwrite: bool = False, preserve_sources: bool """ if preserve_sources and self.base is not None: raise ValueError("preserve_sources requires an unfiltered root table") + self._preflight_portable_persistence() if self.base is not None: materialized = self.copy(compact=True) materialized.save(urlpath, overwrite=overwrite) @@ -12164,6 +12168,8 @@ def _readable_computed_expr(cc: dict) -> str: ``kernel(dep0, dep1)`` call label is returned. """ col_deps = cc["col_deps"] + if cc.get("kind") == "portable": + return f"portable[independent]({', '.join(col_deps)})" if cc.get("kind") == "dsl": kernel = cc.get("kernel") kname = getattr(kernel, "__name__", "dsl_kernel") @@ -12258,7 +12264,19 @@ def _schema_dict_with_computed(self) -> dict: if self._computed_cols: computed = [] for name, cc in self._computed_cols.items(): - if cc.get("kind") == "dsl": + if cc.get("kind") == "portable": + computed.append( + { + "name": name, + "kind": "portable", + "artifact": cc["artifact"], + "bindings": cc["bindings"], + "row_shape": list(cc["row_shape"]), + "col_deps": cc["col_deps"], + "dtype": str(cc["dtype"]), + } + ) + elif cc.get("kind") == "dsl": entry = { "name": name, "kind": "dsl", @@ -12294,6 +12312,8 @@ def _schema_dict_with_computed(self) -> dict: } if "transformer" in meta: entry["transformer"] = meta["transformer"] + if meta.get("transformer_kind") == "portable": + entry.update({key: meta[key] for key in ("artifact", "bindings", "row_shape")}) if meta.get("dsl_source") is not None: entry["dsl_source"] = meta["dsl_source"] if meta.get("jit_backend") is not None: @@ -12488,11 +12508,14 @@ def _load_computed_cols_from_schema(self, schema_dict: dict) -> None: Called from ``__init__``, ``open``, and ``load`` after all stored columns have been opened into ``self._cols``. """ + self._check_legacy_dsl_policy(schema_dict) for cc_meta in schema_dict.get("computed_columns", []): name = cc_meta["name"] col_deps = cc_meta["col_deps"] dtype = np.dtype(cc_meta["dtype"]) - if cc_meta.get("kind") == "dsl": + if cc_meta.get("kind") == "portable": + self._computed_cols[name] = self._validate_portable_table_metadata(cc_meta) + elif cc_meta.get("kind") == "dsl": from blosc2.dsl_kernel import kernel_from_source dsl_source = cc_meta["dsl_source"] @@ -12520,6 +12543,7 @@ def _load_computed_cols_from_schema(self, schema_dict: dict) -> None: def _load_materialized_cols_from_schema(self, schema_dict: dict) -> None: """Reconstruct ``_materialized_cols`` from persisted metadata.""" + self._check_legacy_dsl_policy(schema_dict) for meta in schema_dict.get("materialized_columns", []): loaded = { "computed_column": meta.get("computed_column"), @@ -12531,12 +12555,56 @@ def _load_materialized_cols_from_schema(self, schema_dict: dict) -> None: } if "transformer" in meta: loaded["transformer"] = dict(meta["transformer"]) + if meta.get("transformer_kind") == "portable": + desc = self._validate_portable_table_metadata(meta) + if self._cols[meta["name"]].dtype != desc["dtype"]: + raise ValueError("Stored portable output dtype disagrees with native signature") + loaded.update({key: desc[key] for key in ("artifact", "bindings", "row_shape", "kernel")}) if meta.get("dsl_source") is not None: loaded["dsl_source"] = meta["dsl_source"] if meta.get("jit_backend") is not None: loaded["jit_backend"] = meta["jit_backend"] self._materialized_cols[meta["name"]] = loaded + def _check_legacy_dsl_policy(self, schema_dict): + from blosc2.deserialization import DeserializeMode, normalize_deserialize + from blosc2.exceptions import UnsafeDeserializationError + + legacy = any( + cc.get("kind") == "dsl" or "dsl_source" in cc for cc in schema_dict.get("computed_columns", []) + ) or any( + meta.get("dsl_source") is not None or meta.get("transformer_kind") == "dsl" + for meta in schema_dict.get("materialized_columns", []) + ) + storage = self._storage + policy = getattr( + storage, + "_deserialize_mode", + getattr(getattr(storage, "_store", None), "_deserialize_mode", "safe"), + ) + if legacy and normalize_deserialize(policy) is not DeserializeMode.FULL: + raise UnsafeDeserializationError( + "legacy DSL table columns with block/batch-dependent evaluation", + location="CTable column recipes", + hint="Reopen with deserialize='full' only if you trust this data and " + "intend to retain legacy table-kernel behavior.", + ) + + def _preflight_portable_persistence(self): + for meta in [*self._computed_cols.values(), *self._materialized_cols.values()]: + if meta.get("kind") == "portable" or meta.get("transformer_kind") == "portable": + self._validate_portable_table_metadata(meta) + if any(cc.get("kind") == "dsl" for cc in self._computed_cols.values()) or any( + meta.get("dsl_source") is not None or meta.get("transformer_kind") == "dsl" + for meta in self._materialized_cols.values() + ): + raise TypeError( + "Source-authored vector table columns have no explicit row-domain contract; " + "use add_computed_column or add_generated_column with " + "the DSLKernel, named inputs, and dtype=.... " + "Legacy source metadata is not automatically migrated." + ) + def _require_computed_column(self, name: str) -> dict: """Return metadata for computed column *name* or raise ``KeyError``.""" try: @@ -12560,6 +12628,9 @@ def _autofill_materialized_row_values(self, row: dict[str, Any]) -> dict[str, An if meta.get("transformer_kind") == "row_transformer": transformer = RowTransformer.from_metadata(meta["transformer"]) row[name] = np.asarray(transformer.evaluate_row(row), dtype=meta["dtype"]) + elif meta.get("transformer_kind") == "portable": + single = {dep: [row[dep]] for dep in meta["col_deps"]} + row[name] = self._evaluate_portable_rows(meta, single, count=1)[0] elif meta.get("transformer_kind") == "dsl": single = {dep: [row[dep]] for dep in meta["col_deps"]} row[name] = self._evaluate_dsl_materialized_batch(meta, single)[0] @@ -12607,6 +12678,8 @@ def _autofill_materialized_batch_columns( if meta.get("transformer_kind") == "row_transformer": transformer = RowTransformer.from_metadata(meta["transformer"]) values = transformer.evaluate_batch(raw_columns) + elif meta.get("transformer_kind") == "portable": + values = self._evaluate_portable_rows(meta, raw_columns) elif meta.get("transformer_kind") == "dsl": values = self._evaluate_dsl_materialized_batch(meta, raw_columns) else: @@ -12785,9 +12858,18 @@ def materialize_computed_column( if target_name in self._computed_cols: raise ValueError(f"A computed column named {target_name!r} already exists.") target_dtype = np.dtype(dtype) if dtype is not None else np.dtype(cc["dtype"]) + if cc.get("kind") == "portable" and target_dtype != cc["dtype"]: + raise ValueError("Portable materialization cannot override its native result dtype") self._create_empty_stored_column(target_name, target_dtype, cparams=cparams) - if cc.get("kind") == "dsl": + if cc.get("kind") == "portable": + self._materialized_cols[target_name] = { + **cc, + "computed_column": name, + "transformer_kind": "portable", + "stale": False, + } + elif cc.get("kind") == "dsl": self._materialized_cols[target_name] = { "computed_column": name, "expression": None, @@ -12882,26 +12964,6 @@ def _dsl_deps_from_lazyudf(self, lazyudf) -> list[str]: col_deps.append(cname) return col_deps - def _resolve_dsl_kernel(self, kernel, inputs) -> tuple[Any, list[str]]: - """Validate a bare DSL kernel + its ``inputs`` column bindings.""" - if kernel.dsl_error is not None: - raise blosc2.DSLSyntaxError(f"Invalid DSL kernel: {kernel.dsl_error}") - if inputs is None: - raise TypeError( - "A DSL kernel passed directly requires inputs=[...] naming one source " - "column per kernel parameter." - ) - col_deps = list(inputs) - expected = kernel.input_names - if expected is not None and len(col_deps) != len(expected): - raise ValueError( - f"DSL kernel expects {len(expected)} input(s) {expected}, " - f"but inputs={col_deps} provides {len(col_deps)}." - ) - for d in col_deps: - self._validate_transformer_dep(d) - return kernel, col_deps - def _guard_utf8_kernel_deps(self, col_deps) -> None: """Refuse a UDF/DSL kernel that reads a utf8 column, naming the column. @@ -12927,23 +12989,36 @@ def _guard_utf8_kernel_deps(self, col_deps) -> None: ) ) - def _normalize_transformer(self, expr, inputs=None) -> dict: + def _normalize_transformer(self, expr, inputs=None, *, dtype=None, cardinality="elementwise") -> dict: """Resolve *expr* into a transformer descriptor. Returns one of:: {"kind": "expression", "lazy": , "col_deps": [...]} - {"kind": "dsl", "kernel": , "col_deps": [...]} + {"kind": "portable", "kernel": , "bindings": {...}, ...} A ``@blosc2.dsl_kernel`` is accepted either directly (with *inputs* naming one source column per kernel parameter) or as a ``LazyUDF`` returned by a callable (then operands are matched to columns by identity and *inputs* is ignored). + New kernel registrations always validate/export Menudet and evaluate + independent rows; only loaded legacy recipes use vector-column execution. """ - if isinstance(expr, blosc2.DSLKernel): - kernel, col_deps = self._resolve_dsl_kernel(expr, inputs) - self._guard_utf8_kernel_deps(col_deps) - return {"kind": "dsl", "kernel": kernel, "col_deps": col_deps} + if isinstance(expr, blosc2.DSLKernel | blosc2.PortableKernel): + names = expr.input_names if isinstance(expr, blosc2.DSLKernel) else list(expr.input_dtypes) + if inputs is None: + raise TypeError("A table kernel requires inputs= naming one source column per parameter") + if isinstance(inputs, Mapping): + bindings = dict(inputs) + else: + if len(inputs) != len(names): + raise ValueError(f"Kernel expects {len(names)} inputs, but received {len(inputs)}") + bindings = dict(zip(names, inputs, strict=True)) + self._guard_utf8_kernel_deps(bindings.values()) + if isinstance(expr, blosc2.DSLKernel): + output_dtype = dtype if isinstance(dtype, type) else getattr(dtype, "dtype", dtype) + dtype = self._dsl_result_dtype(expr, list(bindings.values()), output_dtype) + return self._portable_table_descriptor(expr, bindings, dtype=dtype, cardinality=cardinality) # Resolve a callable once (a lambda may return a LazyExpr or a LazyUDF). obj = expr(self._cols) if (callable(expr) and not isinstance(expr, blosc2.LazyExpr)) else expr if isinstance(obj, blosc2.LazyUDF): @@ -12956,12 +13031,12 @@ def _normalize_transformer(self, expr, inputs=None) -> dict: raise blosc2.DSLSyntaxError(f"Invalid DSL kernel: {kernel.dsl_error}") col_deps = self._dsl_deps_from_lazyudf(obj) self._guard_utf8_kernel_deps(col_deps) - return { - "kind": "dsl", - "kernel": kernel, - "col_deps": col_deps, - "jit_backend": obj.kwargs.get("jit_backend"), - } + return self._normalize_transformer( + kernel, + dict(zip(kernel.input_names, col_deps, strict=True)), + dtype=obj.dtype if dtype is None else dtype, + cardinality=cardinality, + ) lazy, col_deps = self._normalize_expression_transformer(obj) # Guard: verify the expression string round-trips before storing. # An empty string means the LazyExpr was not fully constructed, and a @@ -13016,6 +13091,13 @@ def _build_computed_lazy(self, cc: dict): (the all-NDArray miniexpr fast path). The full column is recomputed on each access — acceptable for a virtual, unstored column. """ + if cc.get("kind") == "portable": + positions = np.flatnonzero(self._valid_rows[:]) + raw = {dep: self._cols[dep][positions] for dep in cc["col_deps"]} + values = self._evaluate_portable_rows(cc, raw, count=len(positions)) + output = np.zeros(len(self._valid_rows), dtype=cc["dtype"]) + output[positions] = values + return blosc2.asarray(output) if cc.get("kind") == "dsl": operands = tuple(self._cols[d] for d in cc["col_deps"]) return blosc2.lazyudf( @@ -13082,6 +13164,103 @@ def _generated_dependency_closure(self, source: str) -> set[str]: queue.append(name) return affected + def _portable_table_descriptor(self, kernel, bindings, *, dtype=None, cardinality="elementwise"): + if isinstance(kernel, blosc2.DSLKernel): + if dtype is None: + raise ValueError("Authored portable table kernels require an explicit output dtype") + bindings = dict(bindings) + if any(dep not in self._cols for dep in bindings.values()): + raise ValueError("Portable row inputs require stored columns") + row_shape = ( + np.broadcast_shapes(*(tuple(self._cols[dep].shape[1:]) or (1,) for dep in bindings.values())) + if bindings + else (1,) + ) + kernel = blosc2.PortableKernel.from_json( + kernel.export( + {name: self._cols[dep].dtype for name, dep in bindings.items()}, + dtype, + version="1.0", + ndim=len(row_shape), + cardinality=cardinality, + ), + jit=False, + ) + if not isinstance(kernel, blosc2.PortableKernel): + raise TypeError("Portable table transformers require a validated PortableKernel") + output_dtype = dtype if isinstance(dtype, type) else getattr(dtype, "dtype", dtype) + if dtype is not None and np.dtype(output_dtype) != kernel.output_dtype: + raise ValueError("Output dtype disagrees with the portable kernel signature") + if kernel.schema_version != "1.0": + raise ValueError("Independent rows require portable 1.0") + bindings = dict(bindings) + if set(bindings) != set(kernel.input_dtypes): + raise ValueError("Portable table bindings must match the native signature") + row_shapes = set() + for name, dep in bindings.items(): + if dep not in self._cols: + raise ValueError("Portable row inputs require stored columns") + row_shapes.add(tuple(self._cols[dep].shape[1:]) or (1,)) + if np.dtype(self._cols[dep].dtype) != kernel.input_dtypes[name]: + raise ValueError("Portable table column dtype disagrees with the native signature") + row_shape = np.broadcast_shapes(*row_shapes) if row_shapes else (1,) + if kernel.context_ndim not in (0, len(row_shape)): + raise ValueError("Portable context rank disagrees with the fixed row domain") + if row_shape != (1,) and kernel.result_cardinality != "block_scalar": + raise ValueError("Fixed-shape row transformers currently require scalar native returns") + return { + "kind": "portable", + "kernel": kernel, + "artifact": kernel.to_json(), + "bindings": bindings, + "row_shape": row_shape, + "col_deps": list(dict.fromkeys(bindings.values())), + "dtype": kernel.output_dtype, + } + + def _validate_portable_table_metadata(self, meta): + kernel = blosc2.PortableKernel.from_json(meta["artifact"]) + if meta.get("row_domain", "independent") != "independent": + raise ValueError("Only independent-row portable table groups are implemented") + desc = self._portable_table_descriptor(kernel, meta["bindings"]) + if ( + desc["dtype"] != np.dtype(meta["dtype"]) + or desc["col_deps"] != meta["col_deps"] + or desc["row_shape"] != tuple(meta.get("row_shape", (1,))) + ): + raise ValueError("Portable table metadata disagrees with native signature/bindings") + return desc + + def _evaluate_portable_rows(self, meta, raw_columns, *, count=None): + kernel = meta.get("kernel") + if kernel is None: + kernel = blosc2.PortableKernel.from_json(meta["artifact"]) + arrays = { + name: np.asarray(raw_columns[dep], dtype=kernel.input_dtypes[name]) + for name, dep in meta["bindings"].items() + } + if count is None: + count = len(next(iter(raw_columns.values()))) + if any(len(array) != count for array in arrays.values()): + raise ValueError("Portable row operands disagree on batch length") + result = np.empty(count, dtype=kernel.output_dtype) + row_shape = tuple(meta["row_shape"]) + context = ( + {"logical_shape": row_shape, "block_origin": (0,) * len(row_shape)} + if kernel.context_ndim + else {} + ) + for row in range(count): + # Slice, rather than extract a scalar: NumPy scalar strings discard + # the declared slot width and would invalidate the native signature. + inputs = { + name: np.broadcast_to(array[row : row + 1].reshape(array.shape[1:] or (1,)), row_shape) + for name, array in arrays.items() + } + value = kernel.evaluate_block(inputs, block_shape=row_shape, **context) + result[row] = value[()] if value.shape == () else value[0] + return result + def _mark_generated_columns_stale(self, source: str) -> None: affected = self._generated_dependency_closure(source) changed = False @@ -13104,6 +13283,9 @@ def refresh_generated_column(self, name: str) -> None: if meta.get("transformer_kind") == "row_transformer": transformer = RowTransformer.from_metadata(meta["transformer"]) values = np.asarray(transformer.evaluate_existing(self), dtype=meta["dtype"]) + elif meta.get("transformer_kind") == "portable": + raw_columns = {dep: self[dep][:] for dep in meta["col_deps"]} + values = self._evaluate_portable_rows(meta, raw_columns, count=len(self)) elif meta.get("transformer_kind") == "dsl": raw_columns = {dep: self[dep][:] for dep in meta["col_deps"]} values = self._evaluate_dsl_materialized_batch(meta, raw_columns) @@ -13212,11 +13394,13 @@ def add_generated_column( # noqa: C901 values: str | blosc2.LazyExpr | blosc2.DSLKernel + | blosc2.PortableKernel | Callable[[dict[str, Any]], blosc2.LazyExpr] | RowTransformer, dtype=None, create_index: bool = False, - inputs: list[str] | None = None, + inputs: Mapping[str, str] | list[str] | None = None, + cardinality: str = "elementwise", ) -> None: """Add a stored generated column maintained by the table. @@ -13232,7 +13416,8 @@ def add_generated_column( # noqa: C901 add_generated_column(name, *, values="price * qty", dtype=..., create_index=False) add_generated_column(name, *, values=lazy_expr, dtype=...) - add_generated_column(name, *, values=dsl_kernel, inputs=["price", "qty"], dtype=...) + add_generated_column(name, *, values=dsl_kernel, + inputs={"price": "price", "qty": "qty"}, dtype=...) add_generated_column(name, *, values=blosc2.lazyudf(dsl_kernel, (t.price, t.qty))) add_generated_column(name, *, values=lambda cols: cols["price"] * 1.21, dtype=...) add_generated_column(name, *, values=t.embedding.row_transformer.norm(axis=0), dtype=...) @@ -13253,17 +13438,18 @@ def add_generated_column( # noqa: C901 per row. * :class:`blosc2.LazyExpr`: scalar lazy expression over stored columns of this table. It must produce a 1-D scalar stream. - * :func:`blosc2.dsl_kernel`-decorated kernel passed directly with - ``inputs=[...]`` — one stored scalar column name per kernel - parameter, bound positionally. Produces one scalar per row. - The kernel source is persisted and recompiled on open; appended - rows are auto-filled and :meth:`refresh_generated_column` - recomputes after in-place edits. + * :class:`blosc2.DSLKernel` or :class:`blosc2.PortableKernel` with + named ``inputs={parameter: column}`` bindings (positional lists + are also accepted). Menudet artifacts are validated before + registration and persisted without Python reconstruction. + Each row is an independent logical group, including its complete + fixed-shape array inputs. Appended rows are auto-filled and + :meth:`refresh_generated_column` recomputes after in-place edits. * :class:`blosc2.LazyUDF` built from a :func:`blosc2.dsl_kernel` via :func:`blosc2.lazyudf` — column bindings are inferred by identity from the operands, so ``inputs=`` is not needed. Accepts :class:`Column` accessors (``t.col1``) or raw NDArrays as - operands. Same persistence and auto-fill behaviour as above. + operands. Same independent-row semantics and persistence as above. * callable: called as ``values(self._cols)`` and must return a :class:`blosc2.LazyExpr` or a :class:`blosc2.LazyUDF` backed by a :func:`blosc2.dsl_kernel`. @@ -13274,9 +13460,10 @@ def add_generated_column( # noqa: C901 may produce either one scalar per row or one fixed-shape ndarray item per row. - Expression and DSL forms currently cannot depend on computed columns - and cannot directly consume fixed-shape ndarray columns; use a - row-transformer for ndarray row projections/reductions. + These forms cannot depend on computed columns. Expressions cannot + consume fixed-shape ndarray columns; kernels support scalar row + reductions with ``cardinality="block_scalar"``. Row transformers + also support fixed-shape vector outputs. dtype: Output schema or dtype. Scalar outputs may pass a NumPy dtype or a Blosc2 scalar spec such as ``blosc2.float64()``. Fixed-shape @@ -13291,10 +13478,13 @@ def add_generated_column( # noqa: C901 generated columns raise :class:`ValueError` when indexing is requested. inputs: - Only used when *values* is a bare :func:`blosc2.dsl_kernel`: a list - of stored scalar column names, one per kernel parameter, bound - positionally. Not needed when passing a :class:`blosc2.LazyUDF` or - a callable — bindings are inferred from the operands in those cases. + For bare kernels, a mapping from parameter names to stored columns, + or a positional list of column names. LazyUDF bindings are inferred + from their operands. + cardinality: + Result contract for authored kernels: ``"elementwise"`` by default; + use ``"block_scalar"`` for fixed-shape row reductions. A + :class:`blosc2.PortableKernel` already carries its result contract. Examples -------- @@ -13401,36 +13591,19 @@ def add_generated_column( # noqa: C901 "transformer": transformer.to_metadata(), "stale": False, } - elif (desc := self._normalize_transformer(values, inputs))["kind"] == "dsl": - kernel = desc["kernel"] - col_deps = desc["col_deps"] - compute_dtype = ( - np.dtype(getattr(dtype, "dtype", dtype)) - if dtype is not None - else self._dsl_result_dtype(kernel, col_deps, None) - ) - jit_backend = desc.get("jit_backend") - operands = tuple(self._cols[d] for d in col_deps) - generated_values = np.asarray( - blosc2.lazyudf(kernel, operands, dtype=compute_dtype, jit_backend=jit_backend).compute()[:] - ) - if generated_values.ndim != 1: - raise TypeError("DSL generated columns must produce a 1-D scalar result.") - generated_values = ( - generated_values[self._valid_rows[:]] - if len(generated_values) == len(self._valid_rows) - else generated_values - ) + elif (desc := self._normalize_transformer(values, inputs, dtype=dtype, cardinality=cardinality))[ + "kind" + ] == "portable": + raw = {dep: self[dep][:] for dep in desc["col_deps"]} + generated_values = self._evaluate_portable_rows(desc, raw, count=n_live) spec = self._coerce_generated_spec(dtype, generated_values) metadata = { + **desc, "computed_column": None, "expression": None, - "dsl_source": kernel.dsl_source, - "col_deps": col_deps, "dtype": np.dtype(spec.dtype), - "transformer_kind": "dsl", + "transformer_kind": "portable", "stale": False, - "jit_backend": jit_backend, } else: lazy, col_deps = desc["lazy"], desc["col_deps"] @@ -13480,10 +13653,15 @@ def add_generated_column( # noqa: C901 def add_computed_column( self, name: str, - expr: str | blosc2.LazyExpr | blosc2.DSLKernel | Callable[[dict[str, Any]], blosc2.LazyExpr], + expr: str + | blosc2.LazyExpr + | blosc2.DSLKernel + | blosc2.PortableKernel + | Callable[[dict[str, Any]], blosc2.LazyExpr], *, dtype: np.dtype | None = None, - inputs: list[str] | None = None, + inputs: Mapping[str, str] | list[str] | None = None, + cardinality: str = "elementwise", ) -> None: """Add a read-only virtual column computed from stored columns. @@ -13499,7 +13677,7 @@ def add_computed_column( add_computed_column(name, "price * qty") add_computed_column(name, lazy_expr) - add_computed_column(name, dsl_kernel, inputs=["price", "qty"]) + add_computed_column(name, dsl_kernel, inputs={"price": "price", "qty": "qty"}) add_computed_column(name, blosc2.lazyudf(dsl_kernel, (t.price, t.qty))) add_computed_column(name, lambda cols: cols["price"] * cols["qty"]) @@ -13515,11 +13693,10 @@ def add_computed_column( ``"price * qty"``. * :class:`blosc2.LazyExpr`: lazy expression over stored columns of this table. - * :func:`blosc2.dsl_kernel`-decorated kernel passed directly with - ``inputs=[...]`` — one stored scalar column name per kernel - parameter, bound positionally. The kernel may use loops, - ``if``/``else`` and ``where(...)``. Its source is persisted and - recompiled on open; the column stays virtual/unstored. + * :class:`blosc2.DSLKernel` or :class:`blosc2.PortableKernel` with + named ``inputs={parameter: column}`` bindings (positional lists + are also accepted). Kernels are validated as Menudet before + registration and persist without Python reconstruction. * :class:`blosc2.LazyUDF` built from a :func:`blosc2.dsl_kernel` via :func:`blosc2.lazyudf` — column bindings are inferred by identity from the operands, so ``inputs=`` is not needed. Accepted forms @@ -13529,16 +13706,19 @@ def add_computed_column( :class:`blosc2.LazyExpr` or a :class:`blosc2.LazyUDF` backed by a :func:`blosc2.dsl_kernel`. - DSL columns (last three forms) are persisted — their source is stored - and recompiled on open — and may be referenced inside :meth:`where` - predicates. + Kernel columns use independent-row semantics: scalar inputs form + one-lane groups and fixed-shape inputs supply their complete row. + Reductions never combine different rows. The column remains virtual + and may be referenced inside :meth:`where` predicates. Loaded legacy + table recipes require ``deserialize="full"`` and retain their old + block/batch-dependent semantics, rather than being reinterpreted. Expressions must depend only on stored columns of this table; computed columns cannot depend on other computed columns in this version. Fixed-shape ndarray columns are not accepted in computed - column expressions yet. For row-wise ndarray projections or - reductions, use :meth:`add_generated_column` with - ``values=t.ndarray_col.row_transformer...``. + column expressions yet. Kernels can reduce complete ndarray rows + with ``cardinality="block_scalar"``; generated row transformers + additionally support vector-valued outputs. dtype: Optional dtype override for the computed values. For expression forms it is inferred from the resulting :class:`blosc2.LazyExpr` @@ -13549,11 +13729,13 @@ def add_computed_column( column inputs. This changes the dtype reported by the CTable column wrapper; it does not create physical storage. inputs: - Only used when *expr* is a bare :func:`blosc2.dsl_kernel`: a list of - stored scalar column names, one per kernel parameter, bound - positionally (kernel parameter ``i`` ← ``inputs[i]``). Not needed - when passing a :class:`blosc2.LazyUDF` or a callable — bindings are - inferred from the operands in those cases. + For bare kernels, a mapping from parameter names to stored columns, + or a positional list of column names. LazyUDF bindings are inferred + from their operands. + cardinality: + Result contract for authored kernels: ``"elementwise"`` by default; + use ``"block_scalar"`` for fixed-shape row reductions. A + :class:`blosc2.PortableKernel` already carries its result contract. Examples -------- @@ -13631,18 +13813,9 @@ def ratio(num, den): if name in self._computed_cols: raise ValueError(f"A computed column named {name!r} already exists.") - desc = self._normalize_transformer(expr, inputs) - if desc["kind"] == "dsl": - kernel = desc["kernel"] - col_deps = desc["col_deps"] - self._computed_cols[name] = { - "kind": "dsl", - "dsl_source": kernel.dsl_source, - "kernel": kernel, - "col_deps": col_deps, - "dtype": self._dsl_result_dtype(kernel, col_deps, dtype), - "jit_backend": desc.get("jit_backend"), - } + desc = self._normalize_transformer(expr, inputs, dtype=dtype, cardinality=cardinality) + if desc["kind"] == "portable": + self._computed_cols[name] = desc else: lazy = desc["lazy"] self._computed_cols[name] = { @@ -13777,17 +13950,8 @@ def assign(self, **named_exprs) -> CTable: view._col_widths = dict(self._col_widths) for name, value in bound.items(): desc = view._normalize_transformer(value) - if desc["kind"] == "dsl": - kernel = desc["kernel"] - col_deps = desc["col_deps"] - view._computed_cols[name] = { - "kind": "dsl", - "dsl_source": kernel.dsl_source, - "kernel": kernel, - "col_deps": col_deps, - "dtype": view._dsl_result_dtype(kernel, col_deps, None), - "jit_backend": desc.get("jit_backend"), - } + if desc["kind"] == "portable": + view._computed_cols[name] = desc else: lazy = desc["lazy"] view._computed_cols[name] = { @@ -14893,6 +15057,7 @@ def copy( # noqa: C901 the source. """ if urlpath is not None: + self._preflight_portable_persistence() urlpath = os.fspath(urlpath) if chunks is not None or blocks is not None or cparams is not None: # When storage layout changes we must go through _save_to_storage @@ -15136,18 +15301,10 @@ def _empty_copy( # Rebuild computed columns with the new NDArray objects as operands obj._computed_cols = {} for cc_name, cc in self._computed_cols.items(): - if cc.get("kind") == "dsl": - # DSL entries hold the live kernel; the LazyUDF is rebuilt on - # demand from obj._cols, so no operand rebinding is needed here. - dsl_entry: dict[str, Any] = { - "kind": "dsl", - "dsl_source": cc["dsl_source"], - "kernel": cc["kernel"], - "col_deps": cc["col_deps"], - "dtype": cc["dtype"], - **({"jit_backend": cc["jit_backend"]} if "jit_backend" in cc else {}), - } - obj._computed_cols[cc_name] = dsl_entry + if cc.get("kind") in {"portable", "dsl"}: + # Native/source descriptors resolve columns on demand. Neither + # retains the old table's array operands in a cached expression. + obj._computed_cols[cc_name] = dict(cc) else: operands = {f"o{i}": new_cols[dep] for i, dep in enumerate(cc["col_deps"])} new_lazy = blosc2.lazyexpr(cc["expression"], operands) diff --git a/src/blosc2/deserialization.py b/src/blosc2/deserialization.py index 627163c4e..c71cc4f89 100644 --- a/src/blosc2/deserialization.py +++ b/src/blosc2/deserialization.py @@ -37,6 +37,8 @@ def get_deserialize(obj: Any, default: str | DeserializeMode = DeserializeMode.F """Return the policy attached to *obj* or its physical SChunk carrier.""" carrier = getattr(obj, "schunk", obj) + if carrier is None: + carrier = obj return normalize_deserialize(getattr(carrier, "_deserialize_mode", default)) @@ -45,6 +47,8 @@ def set_deserialize(obj: Any, value: str | DeserializeMode) -> Any: mode = normalize_deserialize(value) carrier = getattr(obj, "schunk", obj) + if carrier is None: + carrier = obj current = getattr(carrier, "_deserialize_mode", None) if current is not None and normalize_deserialize(current) is not mode: raise ValueError( diff --git a/src/blosc2/dict_store.py b/src/blosc2/dict_store.py index b173b6b3b..239d6c388 100644 --- a/src/blosc2/dict_store.py +++ b/src/blosc2/dict_store.py @@ -625,6 +625,9 @@ def __setitem__( self, key: str, value: blosc2.Array | SChunk | blosc2.ObjectArray | blosc2.BatchArray ) -> None: """Add a node to the DictStore.""" + from blosc2.b2objects import preflight_persistence + + preflight_persistence(value) self._modified = True if isinstance(value, np.ndarray): value = blosc2.asarray(value, cparams=self.cparams, dparams=self.dparams) diff --git a/src/blosc2/dsl_artifact_bridge.h b/src/blosc2/dsl_artifact_bridge.h new file mode 100644 index 000000000..df1a95476 --- /dev/null +++ b/src/blosc2/dsl_artifact_bridge.h @@ -0,0 +1,123 @@ +/* Copyright (c) 2026, Blosc Development Team + * SPDX-License-Identifier: BSD-3-Clause + * Optional native artifact support; old miniexpr builds remain usable. */ +#ifndef B2_DSL_ARTIFACT_BRIDGE_H +#define B2_DSL_ARTIFACT_BRIDGE_H +#include "miniexpr.h" +#include +#include + +#ifdef B2_HAVE_PORTABLE_ARTIFACT +#include "miniexpr_artifact.h" +#endif +#if defined(B2_HAVE_PORTABLE_ARTIFACT) && defined(ME_ARTIFACT_DRAFT_SCHEMA_VERSION) +typedef me_artifact_input b2_artifact_input; +typedef me_artifact_error b2_artifact_error; +static int b2_artifact_available(void) { return !strcmp(ME_ARTIFACT_SCHEMA_VERSION, "1.0"); } +static int b2_artifact_load(const char *json, size_t size, int jit, void **out, b2_artifact_error *error) { + if (!b2_artifact_available()) { + *out = NULL; + memset(error, 0, sizeof(*error)); + snprintf(error->message, sizeof(error->message), "Build with miniexpr draft 1.0 artifact support"); + return -100; + } + me_artifact *handle = NULL; + int rc = me_artifact_load(json, size, (me_jit_mode)jit, &handle, error); + *out = handle; + return rc; +} +static int b2_artifact_eval(const void *handle, const b2_artifact_input *inputs, int ninputs, + void *output, size_t count, b2_artifact_error *error) { + return me_artifact_eval((const me_artifact *)handle, inputs, ninputs, output, count, error); +} +static void b2_artifact_free(void *handle) { me_artifact_free((me_artifact *)handle); } +static const char *b2_artifact_source(const void *handle) { return me_artifact_source(handle); } +static const char *b2_artifact_entry(const void *handle) { return me_artifact_entry_point(handle); } +static int b2_artifact_ninputs(const void *handle) { return me_artifact_ninputs(handle); } +static const char *b2_artifact_name(const void *handle, int index) { return me_artifact_input_name(handle, index); } +static me_dtype b2_artifact_dtype(const void *handle, int index) { return me_artifact_input_dtype(handle, index); } +static me_dtype b2_artifact_output(const void *handle) { return me_artifact_output_dtype(handle); } +static int b2_artifact_jit(const void *handle) { return me_artifact_has_jit(handle); } +#else +typedef struct { + const char *name; + me_dtype dtype; + const void *data; + size_t nitems; +} b2_artifact_input; +typedef struct { + int native_status; + int line; + int column; + char message[256]; +} b2_artifact_error; +static int b2_artifact_available(void) { return 0; } +static int b2_artifact_load(const char *json, size_t size, int jit, void **out, b2_artifact_error *error) { + (void)json; (void)size; (void)jit; + *out = NULL; + memset(error, 0, sizeof(*error)); + snprintf(error->message, sizeof(error->message), "Build with miniexpr portable artifact support"); + return -100; +} +static int b2_artifact_eval(const void *handle, const b2_artifact_input *inputs, int ninputs, + void *output, size_t count, b2_artifact_error *error) { + (void)handle; (void)inputs; (void)ninputs; (void)output; (void)count; (void)error; + return -100; +} +static void b2_artifact_free(void *handle) { (void)handle; } +static const char *b2_artifact_source(const void *handle) { (void)handle; return NULL; } +static const char *b2_artifact_entry(const void *handle) { (void)handle; return NULL; } +static int b2_artifact_ninputs(const void *handle) { (void)handle; return 0; } +static const char *b2_artifact_name(const void *handle, int index) { (void)handle; (void)index; return NULL; } +static me_dtype b2_artifact_dtype(const void *handle, int index) { (void)handle; (void)index; return ME_AUTO; } +static me_dtype b2_artifact_output(const void *handle) { (void)handle; return ME_AUTO; } +static int b2_artifact_jit(const void *handle) { (void)handle; return 0; } +#endif +#if defined(B2_HAVE_PORTABLE_ARTIFACT) && defined(ME_ARTIFACT_DRAFT_SCHEMA_VERSION) +typedef me_artifact_buffer b2_artifact_buffer; +typedef me_artifact_eval_descriptor b2_artifact_descriptor; +static int b2_artifact_descriptor_available(void) { return b2_artifact_available(); } +static int b2_artifact_cardinality(const void *handle) { return me_artifact_result_cardinality(handle); } +static size_t b2_artifact_width(const void *handle, int index) { + return index < 0 ? me_artifact_output_itemsize(handle) : me_artifact_input_itemsize(handle, index); +} +static int b2_artifact_rank(const void *handle) { return me_artifact_context_ndim(handle); } +static const char *b2_artifact_version(const void *handle) { return me_artifact_schema_version(handle); } +static int b2_artifact_eval_ex(const void *handle, const b2_artifact_buffer *inputs, int ninputs, + void *output, const b2_artifact_descriptor *descriptor, b2_artifact_error *error) { + return me_artifact_eval_ex(handle, inputs, ninputs, output, descriptor, error); +} +#else +typedef struct { + const char *name; + me_dtype dtype; + size_t itemsize; + const void *data; + size_t capacity; +} b2_artifact_buffer; +typedef struct { + size_t struct_size; + unsigned int version; + size_t nitems; + size_t output_capacity; + const uint8_t *valid_mask; + size_t valid_mask_capacity; + int ndim; + const int64_t *logical_shape; + const int64_t *block_origin; + const int64_t *block_extent; +} b2_artifact_descriptor; +static int b2_artifact_descriptor_available(void) { return 0; } +static int b2_artifact_cardinality(const void *handle) { (void)handle; return 0; } +static size_t b2_artifact_width(const void *handle, int index) { (void)handle; (void)index; return 0; } +static int b2_artifact_rank(const void *handle) { (void)handle; return 0; } +static const char *b2_artifact_version(const void *handle) { (void)handle; return "unsupported"; } +static int b2_artifact_eval_ex(const void *handle, const b2_artifact_buffer *inputs, int ninputs, + void *output, const b2_artifact_descriptor *descriptor, b2_artifact_error *error) { + (void)handle; (void)inputs; (void)ninputs; (void)output; (void)descriptor; + memset(error, 0, sizeof(*error)); + snprintf(error->message, sizeof(error->message), "Build with draft miniexpr 1.0 descriptor support"); + return -2; +} +#endif +#endif diff --git a/src/blosc2/dsl_kernel.py b/src/blosc2/dsl_kernel.py index b19f1965a..6a6226e05 100644 --- a/src/blosc2/dsl_kernel.py +++ b/src/blosc2/dsl_kernel.py @@ -689,7 +689,64 @@ def _cmpop(self, op: ast.cmpop): class DSLKernel: - """Wrap a Python function and optionally extract a miniexpr DSL kernel from it.""" + """Wrap a Python function, or construct a native-only kernel with ``from_source``.""" + + @classmethod + def from_source(cls, source: str) -> DSLKernel: + """Construct a native-only kernel without executing Python source. + + Preserve the source, including header pragmas, verbatim. Only the function + header is inspected here; miniexpr validates the body when compiling with + the input and output dtypes supplied to :func:`blosc2.lazyudf`. + No frontend rewriting, captured globals, or Python fallback is provided. + This accepts native DSL source, not a portable-profile certification. + """ + if not isinstance(source, str): + raise TypeError("DSL source must be a string") + if "\x00" in source: + raise ValueError("Native DSL source must not contain NUL characters") + try: + tokens = list(tokenize.generate_tokens(StringIO(source).readline)) + first = next(t for t in tokens if t.type not in (tokenize.COMMENT, tokenize.NL)) + if first.type != tokenize.NAME or first.string != "def": + raise ValueError("DSL source must start with a function definition") + _, _, colon = _find_def_signature_span(source) + if colon is None: + raise ValueError("DSL source requires a function header") + lines = source.splitlines(keepends=True) + header = "".join(lines[: colon.end[0] - 1]) + lines[colon.end[0] - 1][: colon.end[1]] + function = ast.parse(header + "\n pass\n").body[0] + args = function.args + if ( + args.posonlyargs + or args.vararg + or args.kwarg + or args.kwonlyargs + or args.defaults + or args.kw_defaults + or function.returns + or any(arg.annotation for arg in args.args) + ): + raise ValueError("Native DSL parameters must be plain positional names") + names = [arg.arg for arg in args.args] + if len(set(names)) != len(names): + raise ValueError("Native DSL parameter names must be unique") + except (SyntaxError, tokenize.TokenError, IndentationError, StopIteration) as error: + raise ValueError(f"Invalid native DSL function header: {error}") from error + + kernel = cls.__new__(cls) + kernel.func = None + kernel.__name__ = kernel.__qualname__ = function.name + kernel.__doc__ = None + kernel._sig = None + kernel._sig_has_varargs = False + kernel._sig_npositional = len(names) + kernel._legacy_udf_signature = False + kernel.dsl_source = source + kernel.input_names = names + kernel.dsl_error = None + kernel.row_param = kernel.row_columns = None + return kernel def __init__(self, func): self.func = func @@ -833,12 +890,17 @@ def _rewrite_numpy_attr_calls(func, dsl_source, dsl_tree, dsl_func, input_names) any alias, including a bare `numpy` import, is honored). The DSL grammar only accepts bare function-name calls. No-op, returning the inputs unchanged, when there is nothing to rewrite (including when the alias is - shadowed by one of the kernel's own parameter names). + shadowed by a parameter, local, or closure binding). """ + code = getattr(func, "__code__", None) + shadowed = set(input_names) + if code is not None: + shadowed.update(code.co_varnames) + shadowed.update(code.co_freevars) aliases = { name for name, value in getattr(func, "__globals__", {}).items() - if value is numpy and name not in input_names + if value is numpy and name not in shadowed } if not aliases: return dsl_source, dsl_tree, dsl_func @@ -872,7 +934,51 @@ def _input_names_from_signature(func_node: ast.FunctionDef) -> list[str]: raise ValueError("DSL kernel does not support *args/**kwargs/kwonly args") return [a.arg for a in (args.posonlyargs + args.args)] + def export( + self, + input_dtypes, + output_dtype, + *, + capture_dtypes=None, + constants=None, + metadata=None, + version="1.0", + cardinality="elementwise", + ndim=0, + ): + """Export a typed portable JSON artifact without calling the Python function. + + ``input_dtypes`` maps runtime parameter names to explicit logical dtypes. + Optional ``constants`` supplies scalar values for remaining parameters. + Globals/closure scalars are snapshotted at export and become explicit, + collision-free parameters. ``capture_dtypes`` overrides scalar type + inference (Python bool/int/float infer bool/int64/float64 respectively; + supported NumPy scalars retain their dtype). Arrays, objects, and external + callbacks cannot be captures. Import via :meth:`blosc2.PortableKernel.from_json`. + + The default ``version="1.0"`` is an experimental implementation draft. It + supports mixed numeric types and fixed byte/Unicode string snapshots; + ``cardinality`` is ``elementwise`` or ``block_scalar``, and ``ndim`` + declares the required logical coordinate rank. The native loader + validates normalized source, widths and cardinality before export succeeds. + """ + from .portable_kernel import export_portable_kernel + + return export_portable_kernel( + self, + input_dtypes, + output_dtype, + capture_dtypes=capture_dtypes, + constants=constants, + metadata=metadata, + version=version, + cardinality=cardinality, + ndim=ndim, + ) + def __call__(self, inputs_tuple, output, offset=None): + if self.func is None: + raise RuntimeError("A native-source DSL kernel cannot execute as a Python function") if self.dsl_error is not None: raise self.dsl_error if self._legacy_udf_signature: @@ -962,6 +1068,65 @@ def validate_dsl(func): } +def validate_portable_dsl( + source, input_dtypes, output_dtype, *, language_version="1.0", ndim=0, cardinality=None +): + """Check raw native source and an exact typed signature against a portable profile. + + Return a dictionary with ``valid``, ``status``, ``line``, ``column``, and + ``error``. Validation neither executes a kernel nor invokes a JIT compiler. + It does not normalize frontend syntax, inspect globals, or certify runtime + input ranges, target compatibility, or sandbox safety. Native support is + required; older builds report ``runtime_unsupported`` without Python fallback. + + ``input_dtypes`` maps parameter names to explicit dtype descriptions. Parameter + binding is by name; mapping order may differ from source order. Dtypes describe + logical values, not the storage layout of any actual input arrays. + The default ``language_version="1.0"`` is an experimental implementation + draft admitting mixed numeric and fixed-width ``S``/``U`` + signatures. ``ndim`` declares the logical rank for reserved ND symbols; + missing or insufficient context rejects explicitly. ``cardinality`` may be + ``elementwise`` or ``block_scalar`` to assert the native return contract; + None validates the inferred contract. Neither arrays nor an artifact JSON + loader are needed. Exact string output width is checked natively. + """ + from collections.abc import Mapping + + from . import blosc2_ext + + if not isinstance(source, str) or not isinstance(language_version, str): + raise TypeError("source and language_version must be strings") + if not isinstance(input_dtypes, Mapping): + raise TypeError("input_dtypes must map parameter names to explicit dtypes") + if "\x00" in source or "\x00" in language_version: + raise ValueError("Portable DSL source and version must not contain NUL characters") + if language_version != "1.0": + return { + "valid": False, + "status": "unsupported_version", + "line": 0, + "column": 0, + "error": "Unsupported portable DSL version; expected draft 1.0", + } + if any(not isinstance(name, str) or "\x00" in name for name in input_dtypes): + raise ValueError("Portable DSL input names must be strings without NUL characters") + if type(ndim) is not int or not 0 <= ndim <= 2**31 - 1: + raise ValueError("ndim must be a nonnegative Python integer representable by the native API") + contracts = {None: 0, "elementwise": 1, "block_scalar": 2} + if cardinality not in contracts: + raise ValueError("cardinality must be None, 'elementwise' or 'block_scalar'") + validator = getattr(blosc2_ext, "validate_portable_dsl_source", None) + if validator is None: + return { + "valid": False, + "status": "runtime_unsupported", + "line": 0, + "column": 0, + "error": "Rebuild miniexpr and the extension with portable DSL validation support", + } + return validator(source, input_dtypes, output_dtype, language_version, ndim, contracts[cardinality]) + + def validate_dsl_jit(func, operands, out_dtype, *, shape=(64,), chunks=None, blocks=None): """Report whether a DSL kernel JIT-compiles (vs. interpreter fallback). diff --git a/src/blosc2/embed_store.py b/src/blosc2/embed_store.py index e91abacc7..582a3c265 100644 --- a/src/blosc2/embed_store.py +++ b/src/blosc2/embed_store.py @@ -273,6 +273,9 @@ def __setitem__( self, key: str, value: blosc2.Array | SChunk | blosc2.ObjectArray | blosc2.BatchArray ) -> None: """Add a node to the embed store.""" + from blosc2.b2objects import preflight_persistence + + preflight_persistence(value) if self.mode == "r": raise ValueError("Cannot set items in read-only mode.") with self._write_bracket(): diff --git a/src/blosc2/exceptions.py b/src/blosc2/exceptions.py index 8faae5d52..7854a8a4b 100644 --- a/src/blosc2/exceptions.py +++ b/src/blosc2/exceptions.py @@ -18,12 +18,14 @@ def __init__(self, expr, missing_ops): class UnsafeDeserializationError(ValueError): """Raised when safe deserialization encounters an active serialized value.""" - def __init__(self, kind: str, *, location: str | None = None): + def __init__(self, kind: str, *, location: str | None = None, hint: str | None = None): where = f" in {location}" if location else "" - super().__init__( - f"Encountered active serialized value {kind!r}{where} while using " - "deserialize='safe'. Reopen with deserialize='full' only if you trust " + advice = hint or ( + "Reopen with deserialize='full' only if you trust " "this data and intend to allow reference resolution and lazy-object reconstruction." ) + super().__init__( + f"Encountered active serialized value {kind!r}{where} while using deserialize='safe'. {advice}" + ) self.kind = kind self.location = location diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py index 732c1141e..44945f63f 100644 --- a/src/blosc2/lazyexpr.py +++ b/src/blosc2/lazyexpr.py @@ -14,7 +14,6 @@ import contextlib import copy import enum -import inspect import linecache import math import operator @@ -22,7 +21,6 @@ import pathlib import re import sys -import textwrap import threading from abc import ABC, abstractmethod, abstractproperty from collections.abc import MutableMapping @@ -1530,6 +1528,10 @@ def fill_chunk_operands( def _apply_jit_backend_pragma(expression: str, inputs: dict, jit_backend: str | None) -> str: + # Author-written pragmas are authoritative. Python's synthesized pragma is + # only a local default and must not override the native environment policy. + if "# me:compiler=" in expression: + return expression if jit_backend is None: return expression if jit_backend == "js": @@ -1538,11 +1540,15 @@ def _apply_jit_backend_pragma(expression: str, inputs: dict, jit_backend: str | if jit_backend not in ("tcc", "cc"): raise ValueError("jit_backend must be one of: None, 'tcc', 'cc', 'js'") + env_backend = os.environ.get("ME_DSL_JIT_COMPILER", "").strip().lower() + if env_backend in ("tcc", "cc"): + jit_backend = env_backend + pragma = f"# me:compiler={jit_backend}\n" - stripped = expression.lstrip() + stripped = "\n".join( + line for line in expression.splitlines() if not line.lstrip().startswith("#") + ).lstrip() if stripped.startswith("def "): - if "# me:compiler=" in expression: - return expression return pragma + expression params = ", ".join(k for k, v in inputs.items() if hasattr(v, "dtype")) return f"{pragma}def __me_auto({params}):\n return {expression}" @@ -2733,8 +2739,9 @@ def reduce_slices( # noqa: C901 if ne_args is None: ne_args = {} fp_accuracy = kwargs.pop("fp_accuracy", blosc2.FPAccuracy.DEFAULT) - jit = kwargs.pop("jit", None) - jit_backend = kwargs.pop("jit_backend", None) + # Scalar native reductions cannot use elementwise DSL JIT output buffers. + kwargs.pop("jit", None) + kwargs.pop("jit_backend", None) where: dict | None = kwargs.pop("_where_args", None) reduce_op = reduce_args.pop("op") reduce_op_str = reduce_args.pop("op_str", None) @@ -2912,8 +2919,13 @@ def reduce_slices( # noqa: C901 expression_miniexpr = f"{reduce_op_str}(where({expression}, _where_x, _where_y))" else: expression_miniexpr = f"{reduce_op_str}({expression})" - expression_miniexpr = _apply_jit_backend_pragma(expression_miniexpr, operands, jit_backend) - res_eval._set_pref_expr(expression_miniexpr, operands, fp_accuracy, aux_reduc, jit=jit) + # Keep reductions as native scalar expressions. A DSL return + # broadcasts a reduced value across the output block, whereas this + # prefilter supplies only one accumulator slot per block. Explicit + # JIT_ON also auto-lifts plain expressions in the native compiler, + # so use the scalar interpreter until reduction JIT has a separate + # output-cardinality contract. + res_eval._set_pref_expr(expression_miniexpr, operands, fp_accuracy, aux_reduc, jit=False) prefilter_set = True # print("expr->miniexpr:", expression, reduce_op, fp_accuracy) # Data won't even try to be compressed, so buffers can be unitialized and reused @@ -5092,6 +5104,10 @@ def __init__( def dtype(self): return self._dtype + @property + def nbytes(self): + return math.prod(self.shape) * np.dtype(self.dtype).itemsize + @property def ndim(self) -> int: return len(self.shape) @@ -5282,65 +5298,20 @@ def save(self, urlpath=None, **kwargs): ----- * All operands must be :ref:`NDArray` or :ref:`C2Array` objects stored on disk or a remote server (i.e. they must have a ``urlpath``). - * When the :ref:`LazyUDF` wraps a :func:`blosc2.dsl_kernel`-decorated - function, the DSL source is preserved verbatim in the saved metadata. - On reload via :func:`blosc2.open`, the function is restored as a full - :class:`~blosc2.dsl_kernel.DSLKernel` so the miniexpr JIT fast path - remains available without any extra work from the caller. + * Newly authored DSLKernel functions normalize and validate as portable + 1.0 before saving. Python UDFs and noncompliant kernels reject before + writing. Block-scalar kernels require the explicit portable API. + Historical source-backed files require ``deserialize='full'`` on open + and are not automatically migrated by saving. """ if urlpath is None: raise ValueError("To save a LazyArray you must provide an urlpath") - kwargs["urlpath"] = urlpath - kwargs["mode"] = "w" # always overwrite the file in urlpath - try: - self._to_b2object_carrier(**kwargs) - except (TypeError, ValueError): - meta = kwargs.get("meta", {}) - meta["LazyArray"] = LazyArrayEnum.UDF.value - kwargs["meta"] = meta - - # Create an empty array; useful for providing the shape and dtype of the outcome - array = blosc2.empty(shape=self.shape, dtype=self.dtype, **kwargs) + from blosc2.portable_lazy import portable_from_lazyudf - # Save the expression and operands in the metadata - operands = {} - operands_ = self.inputs_dict - for i, (_key, value) in enumerate(operands_.items()): - pos_key = f"o{i}" # always use positional keys for consistent loading - if isinstance(value, blosc2.C2Array): - operands[pos_key] = { - "path": str(value.path), - "urlbase": value.urlbase, - } - continue - if isinstance(value, blosc2.Proxy): - # Take the required info from the Proxy._cache container - value = value._cache - if not hasattr(value, "schunk"): - raise ValueError( - "To save a LazyArray, all operands must be blosc2.NDArray or blosc2.C2Array objects" - ) from None - if value.schunk.urlpath is None: - raise ValueError( - "To save a LazyArray, all operands must be stored on disk/network" - ) from None - operands[pos_key] = value.schunk.urlpath - udf_func = self.func.func if isinstance(self.func, DSLKernel) else self.func - udf_name = getattr(udf_func, "__name__", self.func.__name__) - try: - udf_source = textwrap.dedent(inspect.getsource(udf_func)).lstrip() - except Exception: - udf_source = None - meta = { - "UDF": udf_source, - "operands": operands, - "name": udf_name, - } - if isinstance(self.func, DSLKernel) and self.func.dsl_source is not None: - meta["dsl_source"] = self.func.dsl_source - array.schunk.vlmeta["_LazyArray"] = meta - write_b2object_user_vlmeta(array, self._get_user_vlmeta()) + portable = portable_from_lazyudf(self) + portable.save(urlpath, **kwargs) + self.array, self.schunk = portable.array, portable.schunk def to_cframe(self) -> bytes: return self._to_b2object_carrier().to_cframe() @@ -5367,8 +5338,12 @@ def _to_b2object_carrier(self, **kwargs): ref_urlpath = operand_urlpath.as_posix() operand_payload["urlpath"] = ref_urlpath + from blosc2.msgpack_utils import msgpack_packb + + msgpack_packb(payload) + msgpack_packb(self._get_user_vlmeta()) array = make_b2object_carrier( - "lazyudf", + "portable", self.shape, self.dtype, chunks=self.chunks, @@ -5738,6 +5713,12 @@ def _reconstruct_lazyudf(expr, lazyarray, operands_dict, array): def open_lazyarray(array): + from blosc2.deserialization import DeserializeMode, get_deserialize + from blosc2.exceptions import UnsafeDeserializationError + + policy = get_deserialize(array) + if policy is not DeserializeMode.FULL: + raise UnsafeDeserializationError("legacy LazyArray") value = array.schunk.meta["LazyArray"] lazyarray = array.schunk.vlmeta["_LazyArray"] if value == LazyArrayEnum.Expr.value: @@ -5755,7 +5736,7 @@ def open_lazyarray(array): if isinstance(v, str): v = parent_path / v try: - op = blosc2.open(v, mode="r") + op = blosc2.open(v, mode="r", deserialize=policy) except FileNotFoundError: missing_ops[key] = v else: @@ -5780,6 +5761,7 @@ def open_lazyarray(array): new_expr = LazyExpr._new_expr(expr, operands_dict, guess=True, out=None, where=None) elif value == LazyArrayEnum.UDF.value: new_expr = _reconstruct_lazyudf(expr, lazyarray, operands_dict, array) + new_expr._legacy_source_recipe = True # Make the array info available for the user (only available when opened from disk) new_expr.array = array diff --git a/src/blosc2/msgpack_utils.py b/src/blosc2/msgpack_utils.py index b95890fcf..2854bb0ab 100644 --- a/src/blosc2/msgpack_utils.py +++ b/src/blosc2/msgpack_utils.py @@ -48,7 +48,7 @@ def _encode_structured_reference(obj): return None -def _decode_structured_reference(data): +def _decode_structured_reference(data, *, deserialize="safe"): payload = unpackb(data) if not isinstance(payload, dict): raise TypeError("Structured Blosc2 msgpack payload must decode to a mapping") @@ -61,8 +61,8 @@ def _decode_structured_reference(data): if kind == "ref": ref_payload = payload.get("ref") return Ref.from_dict(ref_payload) - if kind in {"c2array", "lazyexpr", "lazyudf"}: - return decode_b2object_payload(payload) + if kind in {"c2array", "lazyexpr", "lazyudf", "portable"}: + return decode_b2object_payload(payload, deserialize=deserialize) raise ValueError(f"Unsupported structured Blosc2 msgpack payload kind: {kind!r}") @@ -80,7 +80,7 @@ def _encode_ndarray(value): return ExtType(_BLOSC2_NDARRAY_EXT_CODE, msgpack_packb(payload)) -def _decode_ndarray(data, *, deserialize=DeserializeMode.FULL): +def _decode_ndarray(data, *, deserialize=DeserializeMode.SAFE): from blosc2.hdf5_source import dtype_from_value payload = msgpack_unpackb(data, deserialize=deserialize) @@ -139,34 +139,38 @@ def decode_tuple_list_hook(obj): return obj -def _decode_msgpack_ext(code, data): +def _decode_msgpack_ext(code, data, *, deserialize=DeserializeMode.SAFE): import blosc2 if code == _BLOSC2_EXT_CODE: - return blosc2.from_cframe(data, copy=True, deserialize="full") + return blosc2.from_cframe(data, copy=True, deserialize=deserialize) if code == _BLOSC2_STRUCTURED_EXT_CODE: - return _decode_structured_reference(data) + return _decode_structured_reference(data, deserialize=deserialize) if code == _BLOSC2_COMPLEX_EXT_CODE: real, imag = struct.unpack(">dd", data) return complex(real, imag) if code == _BLOSC2_NDARRAY_EXT_CODE: - return _decode_ndarray(data, deserialize=DeserializeMode.FULL) + return _decode_ndarray(data, deserialize=deserialize) if code == _BLOSC2_SET_EXT_CODE: - return set(msgpack_unpackb(data)) + return set(msgpack_unpackb(data, deserialize=deserialize)) return ExtType(code, data) -def msgpack_unpackb(payload, *, deserialize=DeserializeMode.FULL): +def msgpack_unpackb(payload, *, deserialize=DeserializeMode.SAFE): """Decode a Blosc2 MessagePack value under the selected policy. - This low-level helper retains ``"full"`` as its default for compatibility. - APIs accepting persisted input pass their safe-by-default policy explicitly. + Active reconstruction requires explicit ``deserialize="full"``. Container + readers pass their effective policy through recursive values unchanged. """ mode = normalize_deserialize(deserialize) if mode is DeserializeMode.SAFE: return _safe_msgpack_unpackb(payload) - return unpackb(payload, list_hook=decode_tuple_list_hook, ext_hook=_decode_msgpack_ext) + return unpackb( + payload, + list_hook=decode_tuple_list_hook, + ext_hook=lambda code, data: _decode_msgpack_ext(code, data, deserialize=mode), + ) def _safe_msgpack_unpackb(payload): @@ -205,12 +209,15 @@ def decode_ext(code, data): if code == _BLOSC2_EXT_CODE: kind = "embedded cframe" elif code == _BLOSC2_STRUCTURED_EXT_CODE: + candidate = None try: envelope = unpackb(data) candidate = envelope.get("kind") if isinstance(envelope, dict) else None kind = candidate if isinstance(candidate, str) else "structured reference" except Exception: kind = "structured reference" + if candidate in {"portable", "lazyexpr"}: + return decode_b2object_payload(envelope, deserialize="safe") else: kind = f"MessagePack extension code {code}" raise UnsafeDeserializationError(str(kind)) diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index 98ab50b0a..514a1a292 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -5003,6 +5003,9 @@ def to_cframe(self) -> bytes: >>> print("Data type of the NDArray:", blosc_array.dtype) Data type of the NDArray: int32 """ + from blosc2.b2objects import preflight_persistence + + preflight_persistence(self) return super().to_cframe() def copy(self, dtype: np.dtype | str = None, **kwargs: Any) -> NDArray: @@ -5097,6 +5100,9 @@ def save(self, urlpath: str, contiguous=True, **kwargs: Any) -> None: >>> # Save the array to a file >>> a.save("array.b2frame") """ + from blosc2.b2objects import preflight_persistence + + preflight_persistence(self) urlpath = normalize_urlpath(urlpath) if is_fsspec_url(urlpath): if not contiguous: diff --git a/src/blosc2/portable_kernel.py b/src/blosc2/portable_kernel.py new file mode 100644 index 000000000..b55651674 --- /dev/null +++ b/src/blosc2/portable_kernel.py @@ -0,0 +1,465 @@ +####################################################################### +# Copyright (c) 2026, Blosc Development Team +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### +"""Portable DSL authoring and native-only execution (draft descriptors when installed).""" + +from __future__ import annotations + +import ast +import builtins +import json +import math +import tokenize +from collections.abc import Mapping +from io import StringIO +from types import FunctionType, MappingProxyType + +import numpy as np + + +class PortableArtifactError(ValueError): + """Artifact failure with a stable category and optional native/source diagnostics.""" + + def __init__(self, message, *, status, native_status=0, line=0, column=0): + self.status = status + self.native_status = native_status + self.line = line + self.column = column + location = f" at line {line}, column {column}" if line else "" + super().__init__(f"{status}{location}: {message}") + + +def _mapping(value, label): + if not isinstance(value, Mapping): + raise TypeError(f"{label} must be a mapping") + if any(not isinstance(name, str) or not name or "\x00" in name for name in value): + raise ValueError(f"{label} names must be nonempty strings without NUL characters") + return dict(value) + + +def portable_dtype_descriptor(value): + dtype = np.dtype(value).newbyteorder("=") + if ( + dtype.fields + or dtype.subdtype + or dtype.kind not in "biufSU" + or (dtype.kind == "f" and dtype.itemsize not in (4, 8)) + ): + raise PortableArtifactError(f"Unsupported portable dtype {dtype}", status="unsupported_requirement") + if dtype.kind in "SU": + if not dtype.itemsize: + raise PortableArtifactError("Fixed strings require a positive width", status="binding_error") + return {"dtype": "bytes" if dtype.kind == "S" else "unicode32", "itemsize": dtype.itemsize} + return {"dtype": dtype.name} + + +def portable_scalar_descriptor(name, value, dtype=None): + numeric = ( + bool, + int, + float, + np.bool_, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + np.float32, + np.float64, + ) + if type(value) not in numeric + (str, bytes, np.str_, np.bytes_): + raise PortableArtifactError("Captures must be plain numeric/string scalars", status="binding_error") + if dtype is None: + dtype = {bool: "bool", int: "int64", float: "float64"}.get(type(value), np.asarray(value).dtype) + descriptor = portable_dtype_descriptor(dtype) + dtype = np.dtype(dtype).newbyteorder("=") + if dtype.kind in "SU": + if (dtype.kind == "S" and type(value) not in (bytes, np.bytes_)) or ( + dtype.kind == "U" and type(value) not in (str, np.str_) + ): + raise PortableArtifactError("String capture family mismatch", status="binding_error") + if len(value) > dtype.itemsize // (4 if dtype.kind == "U" else 1): + raise PortableArtifactError("String capture exceeds its fixed width", status="binding_error") + if dtype.kind == "U" and any(0xD800 <= ord(c) <= 0xDFFF for c in value): + raise PortableArtifactError("Invalid Unicode scalar", status="binding_error") + encoded = np.asarray(value, dtype=dtype).astype(dtype.newbyteorder(">"), copy=False).tobytes().hex() + return { + "name": name, + **descriptor, + "encoding": "bytes-hex" if dtype.kind == "S" else "unicode32be-hex", + "value": encoded, + } + if dtype.kind in "biu": + if type(value) in (float, np.float32, np.float64, str, bytes, np.str_, np.bytes_): + raise PortableArtifactError("Integral captures require integral values", status="binding_error") + integer = int(value) + lower, upper = (0, 1) if dtype.kind == "b" else (np.iinfo(dtype).min, np.iinfo(dtype).max) + if not lower <= integer <= upper: + raise PortableArtifactError("Capture is outside its dtype", status="binding_error") + return { + "name": name, + **descriptor, + "encoding": "boolean" if dtype.kind == "b" else "decimal", + "value": bool(integer) if dtype.kind == "b" else str(integer), + } + if type(value) in (str, bytes, np.str_, np.bytes_): + raise PortableArtifactError("Numeric capture family mismatch", status="binding_error") + integral = type(value) in ( + bool, + int, + np.bool_, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + ) + # NumPy signed minima cannot be negated in their own fixed-width dtype. + magnitude = abs(int(value)) if integral else abs(float(value)) + if (integral or math.isfinite(value)) and magnitude > float(np.finfo(dtype).max): + raise PortableArtifactError("Floating capture overflows its dtype", status="binding_error") + scalar = np.asarray(value, dtype=dtype) + return { + "name": name, + **descriptor, + "encoding": "ieee754-hex", + "value": scalar.astype(dtype.newbyteorder(">"), copy=False).tobytes().hex(), + } + + +def _parameterize_source(source, replacements, parameters): + """Token edits retain comments/pragmas and never edit strings or substrings.""" + tokens = list(tokenize.generate_tokens(StringIO(source).readline)) + starts = [0] + for line in source.splitlines(keepends=True): + starts.append(starts[-1] + len(line)) + + def offset(position): + return starts[position[0] - 1] + position[1] + + edits = [] + depth = 0 + signature = False + for token in tokens: + if token.type == tokenize.NAME and token.string == "def": + signature = True + if signature and token.string == "(": + depth += 1 + elif signature and token.string == ")": + depth -= 1 + if depth == 0: + if parameters: + edits.append((offset(token.start), offset(token.start), ", ".join(parameters))) + signature = False + elif token.type == tokenize.NAME and token.string in replacements and not signature: + edits.append((offset(token.start), offset(token.end), replacements[token.string])) + for start, end, text in sorted(edits, reverse=True): + source = source[:start] + text + source[end:] + return source + + +def _capture_scope(func): + if not isinstance(func, FunctionType): + raise PortableArtifactError( + "Only ordinary Python functions can export captures", status="unsupported_requirement" + ) + scope = func.__globals__.copy() + for name, cell in zip(func.__code__.co_freevars, func.__closure__ or (), strict=True): + try: + scope[name] = cell.cell_contents + except ValueError as error: + raise PortableArtifactError(f"Unbound capture {name!r}", status="invalid_source") from error + return scope + + +def _export_captures(func, source, names, capture_dtypes): + scope = _capture_scope(func) + tree = ast.parse(source) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef)) + nodes = list(ast.walk(function)) + local = set(names) | { + node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) + } + calls = { + node.func.id for node in nodes if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + } + # Native compilation owns builtin membership. Never export external callbacks. + if any(isinstance(n, (ast.Attribute, ast.Subscript, ast.Lambda)) for n in nodes): + raise PortableArtifactError("Unsupported dynamic authoring syntax", status="unsupported_requirement") + for call in calls: + allowed = (getattr(builtins, call, None), getattr(np, call, None), getattr(math, call, None)) + if call in local or (call in scope and not any(v is not None and scope[call] is v for v in allowed)): + raise PortableArtifactError( + f"External callback {call!r} cannot be exported", status="unsupported_requirement" + ) + loads = {node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)} + occupied = local | loads | {function.name} + replacements = {} + encoded = [] + for name in sorted(loads - local - calls): + if name in {"_flat_idx", "_ndim"} or (name.startswith(("_i", "_n")) and name[2:].isdigit()): + continue + if name not in scope: + node = next(node for node in nodes if isinstance(node, ast.Name) and node.id == name) + raise PortableArtifactError( + f"Unresolved capture {name!r}", + status="invalid_source", + line=node.lineno, + column=node.col_offset + 1, + ) + index = len(replacements) + candidate = f"_capture_{index}" + while candidate in occupied: + index += 1 + candidate = f"_capture_{index}" + occupied.add(candidate) + scalar = portable_scalar_descriptor(name, scope[name], capture_dtypes.get(name)) + scalar["name"] = candidate + encoded.append(scalar) + replacements[name] = candidate + added = list(replacements.values()) + if names and added: + added[0] = ", " + added[0] + return ( + _parameterize_source(source, replacements, added), + encoded, + set(replacements) & set(capture_dtypes), + ) + + +def export_portable_kernel( + kernel, + input_dtypes, + output_dtype, + *, + capture_dtypes=None, + constants=None, + metadata=None, + version="1.0", + cardinality="elementwise", + ndim=0, +): + """Implementation of :meth:`blosc2.DSLKernel.export`; does not call the kernel.""" + if version != "1.0": + raise PortableArtifactError("Unsupported artifact version", status="unsupported_version") + inputs = _mapping(input_dtypes, "input_dtypes") + constants = _mapping({} if constants is None else constants, "constants") + captures = _mapping({} if capture_dtypes is None else capture_dtypes, "capture_dtypes") + source = kernel.dsl_source + if not isinstance(source, str): + raise PortableArtifactError("Kernel has no normalized source", status="invalid_source") + names = kernel.input_names + if set(inputs) & set(constants) or set(inputs) | set(constants) != set(names): + raise PortableArtifactError( + "Every parameter needs exactly one input or constant binding", status="binding_error" + ) + encoded = [ + portable_scalar_descriptor(name, value, captures.get(name)) for name, value in constants.items() + ] + used = set(constants) & set(captures) + if kernel.func is not None: + source, captured, extra = _export_captures(kernel.func, source, names, captures) + encoded.extend(captured) + used |= extra + if set(captures) - used: + raise PortableArtifactError("Unused capture dtype names", status="binding_error") + if cardinality not in {"elementwise", "block_scalar"} or type(ndim) is not int or ndim < 0: + raise PortableArtifactError("Invalid return/context contract", status="binding_error") + if metadata is not None and not isinstance(metadata, Mapping): + raise TypeError("metadata must be a mapping") + # This finite superset grants no unsupported operations: native typed import + # validates the entire normalized source before this export can succeed. + manifest = { + "schema_version": "1.0", + "language": {"name": "miniexpr", "version": "1.0"}, + "requires": ["numeric", "control-flow", "block-reductions", "fixed-strings", "nd-context"], + "source": source, + "entry_point": kernel.__name__, + "inputs": [ + {"name": name, **portable_dtype_descriptor(inputs[name])} for name in names if name in inputs + ], + "constants": sorted(encoded, key=lambda item: item["name"]), + "output": {**portable_dtype_descriptor(output_dtype), "contract": cardinality}, + "semantics": {"fp": "strict"}, + "context": {"ndim": ndim}, + "metadata": dict(metadata or {}), + } + artifact = ( + json.dumps(manifest, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + + "\n" + ) + PortableKernel.from_json(artifact, jit=False) + return artifact + + +class PortableKernel: + """An owned, typed native artifact; no originating function or Python fallback. + + Use ``from_json`` to import and ``evaluate`` to return NumPy values. Inputs + are same-shaped logical arrays, bound by name. The installed native adapter + enforces its supported profiles. Draft 1.0 descriptor blocks are available + only when that runtime exposes the extended ABI; no sibling build is inferred. + """ + + @classmethod + def from_json(cls, artifact, *, jit=None): + from . import blosc2_ext + + if not isinstance(artifact, str | bytes): + raise TypeError("artifact must be JSON text or UTF-8 bytes, not a path or decoded mapping") + if jit is not None and type(jit) is not bool: + raise TypeError("jit must be True, False, or None") + handle_type = getattr(blosc2_ext, "PortableArtifactHandle", None) + if handle_type is None: + raise NotImplementedError("Build with miniexpr portable artifact support") + try: + data = artifact.encode("utf-8") if isinstance(artifact, str) else artifact + except UnicodeEncodeError as error: + raise PortableArtifactError( + "Artifact text must be valid UTF-8", status="invalid_artifact" + ) from error + handle = handle_type(data, 0 if jit is None else 1 if jit else 2) + instance = cls.__new__(cls) + instance._handle = handle + instance._artifact = data + instance._info = handle.info() + if getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(): + instance._info.update(handle.descriptor_info()) + return instance + + @property + def source(self): + return self._info["source"] + + @property + def entry_point(self): + return self._info["entry_point"] + + @property + def input_dtypes(self): + return MappingProxyType(self._info["inputs"]) + + @property + def output_dtype(self): + return self._info["output_dtype"] + + @property + def has_jit(self): + return self._info["jit"] + + @property + def schema_version(self): + return self._info.get("schema_version", "1.0") + + @property + def result_cardinality(self): + return self._info.get("cardinality", "elementwise") + + @property + def context_ndim(self): + return self._info.get("ndim", 0) + + def to_json(self): + """Return the original validated JSON text, without dropping metadata.""" + return self._artifact.decode("utf-8") + + def evaluate(self, inputs, *, shape=None): + """Evaluate named arrays into a new NumPy array, preserving their common shape. + + No implicit dtype conversion or array broadcasting is allowed. Host endian, + alignment, and strides are normalized into temporary contiguous buffers. + Constant-only kernels require an explicit output ``shape`` (possibly empty). + Draft block-scalar kernels return a scalar NumPy array. ND kernels require + ``evaluate_block`` with explicit logical context. + """ + inputs = _mapping(inputs, "inputs") + if set(inputs) != set(self.input_dtypes): + raise PortableArtifactError("Missing or extra runtime inputs", status="binding_error") + if shape is not None: + if isinstance(shape, int): + shape = (shape,) + else: + shape = tuple(shape) + if any(type(size) is not int or size < 0 for size in shape): + raise ValueError("shape must contain nonnegative Python integers") + arrays = {} + for name, value in inputs.items(): + array = np.asarray(value) + dtype = self.input_dtypes[name] + if array.dtype.newbyteorder("=") != dtype: + raise PortableArtifactError( + f"Input {name!r} must have dtype {dtype}, not {array.dtype}", status="binding_error" + ) + if shape is None: + shape = array.shape + if array.shape != shape: + raise PortableArtifactError( + "All input shapes must match the output shape", status="binding_error" + ) + arrays[name] = np.require(array, dtype=dtype, requirements=["C", "A"]) + if shape is None: + raise ValueError("Constant-only kernels require an explicit shape") + if self.context_ndim: + raise ValueError( + "Use evaluate_block with explicit logical_shape and block_origin for ND artifacts" + ) + return self._handle.evaluate_block(arrays, shape) + + def evaluate_block( + self, inputs, *, block_shape=None, logical_shape=None, block_origin=None, valid_mask=None + ): + """Evaluate one explicit draft 1.0 block, returning its declared cardinality. + + This is standalone block execution, not lazy block-grid scheduling. Native + validation owns coordinates, participating masks and result allocation. + """ + if self.schema_version != "1.0": + raise NotImplementedError("Explicit descriptor blocks require an installed draft 1.0 runtime") + inputs = _mapping(inputs, "inputs") + if set(inputs) != set(self.input_dtypes): + raise PortableArtifactError("Missing or extra runtime inputs", status="binding_error") + if isinstance(block_shape, int): + block_shape = (block_shape,) + elif block_shape is not None: + block_shape = tuple(block_shape) + if block_shape is not None and any(type(size) is not int or size < 0 for size in block_shape): + raise ValueError("block_shape must contain nonnegative Python integers") + arrays = {} + for name, value in inputs.items(): + array = np.asarray(value) + dtype = self.input_dtypes[name] + if array.dtype.newbyteorder("=") != dtype: + raise PortableArtifactError( + f"Input {name!r} must have dtype {dtype}, not {array.dtype}", status="binding_error" + ) + if block_shape is None: + block_shape = array.shape + if array.shape != block_shape: + raise PortableArtifactError( + "Input shape disagrees with block extent", status="binding_error" + ) + arrays[name] = np.require(array, dtype=dtype, requirements=["C", "A"]) + if block_shape is None: + raise ValueError("Constant-only kernels require an explicit block_shape") + return self._handle.evaluate_block( + arrays, + block_shape, + logical_shape=logical_shape, + block_origin=block_origin, + valid_mask=valid_mask, + ) + + def lazy(self, inputs, *, shape=None, partitions=None): + """Bind native inputs to an immutable logical block grid, not a storage grid.""" + from .portable_lazy import PortableLazyArray + + if self.schema_version != "1.0": + raise NotImplementedError("Logical portable partitions require the descriptor profile") + return PortableLazyArray(self, inputs, shape=shape, partitions=partitions) diff --git a/src/blosc2/portable_lazy.py b/src/blosc2/portable_lazy.py new file mode 100644 index 000000000..dc7dcaef4 --- /dev/null +++ b/src/blosc2/portable_lazy.py @@ -0,0 +1,257 @@ +"""Native portable kernels over immutable logical partitions, independent of storage tiles.""" + +from itertools import product +from math import prod + +import numpy as np + +import blosc2 +from blosc2.lazyexpr import LazyArray + + +def portable_from_lazyudf(obj): + """Preflight a newly authored DSL recipe without executing it or touching storage.""" + from blosc2.dsl_kernel import DSLKernel + from blosc2.portable_kernel import PortableArtifactError, PortableKernel + + if not isinstance(obj.func, DSLKernel): + raise TypeError("Python LazyUDF cannot be persisted; use a compliant DSLKernel or PortableKernel") + schunk = getattr(obj, "schunk", None) + if getattr(obj, "_legacy_source_recipe", False) or ( + schunk is not None + and (schunk.meta.get("LazyArray") is not None or schunk.meta.get("b2o", {}).get("kind") == "lazyudf") + ): + raise TypeError("Persisting legacy source-backed LazyUDF is unsupported; author a new DSLKernel") + inputs, constants = {}, {} + for name, value in obj.inputs_dict.items(): + if np.isscalar(value): + constants[name] = value + else: + inputs[name] = value + shape = tuple(int(n) for n in obj.shape) + partitions = tuple(int(n) for n in obj.blocks) + chunks = tuple(int(n) for n in obj.chunks) + if any(c % b for c, b in zip(chunks, partitions, strict=True)): + raise ValueError( + "LazyUDF chunk boundaries split its logical block grid; use PortableKernel.lazy " + "with explicit partitions" + ) + try: + artifact = obj.func.export( + {name: value.dtype for name, value in inputs.items()}, + obj.dtype, + version="1.0", + constants=constants, + ndim=len(shape), + cardinality="elementwise", + ) + except PortableArtifactError as error: + if "return cardinality" in str(error): + raise ValueError( + "LazyUDF declares elementwise shape but the kernel returns a block scalar; " + "use DSLKernel.export(cardinality='block_scalar', version='1.0') and " + "PortableKernel.lazy with explicit partitions" + ) from error + raise + portable = PortableLazyArray( + PortableKernel.from_json(artifact, jit=False), inputs, shape=shape, partitions=partitions + ) + portable._set_user_vlmeta(obj._get_user_vlmeta(), sync=False) + return portable + + +class PortableLazyArray(LazyArray): + def __init__(self, kernel, inputs, *, shape=None, partitions=None): + self.kernel = kernel + self.inputs = dict(inputs) + if set(self.inputs) != set(kernel.input_dtypes): + raise ValueError("Portable input names disagree with native signature") + if shape is None: + if not self.inputs: + raise ValueError("Constant-only portable arrays require shape") + shape = np.broadcast_shapes(*(value.shape for value in self.inputs.values())) + self._domain = tuple(shape) + if any(type(n) is not int or not 0 <= n <= np.iinfo(np.int64).max for n in self._domain): + raise ValueError("Logical shape must contain nonnegative integers") + if kernel.context_ndim and kernel.context_ndim != len(self._domain): + raise ValueError("Artifact context rank disagrees with logical domain") + for name, value in self.inputs.items(): + if ( + np.broadcast_shapes(tuple(value.shape), self._domain) != self._domain + or np.dtype(value.dtype).newbyteorder("=") != kernel.input_dtypes[name] + ): + raise ValueError( + "Portable operands must broadcast to the logical domain and match native dtype" + ) + self._partitions = tuple(partitions if partitions is not None else (max(n, 1) for n in self._domain)) + if len(self._partitions) != len(self._domain) or any( + type(n) is not int or n <= 0 for n in self._partitions + ): + raise ValueError("Logical partitions must have matching rank and positive extents") + if prod(self._partitions) > 2147483647: + raise ValueError("Logical partition exceeds the native lane limit") + self._grid = tuple((n + p - 1) // p for n, p in zip(self._domain, self._partitions, strict=True)) + self._shape = self._grid if kernel.result_cardinality == "block_scalar" else self._domain + + @property + def shape(self): + return self._shape + + @property + def dtype(self): + return self.kernel.output_dtype + + @property + def nbytes(self): + return prod(self.shape) * self.dtype.itemsize + + @property + def partitions(self): + return self._partitions + + @property + def chunks(self): + return tuple(max(n, 1) for n in self.shape) + + @property + def blocks(self): + return self.chunks + + def _values(self, selection=None): + if selection is None: + selection = tuple(slice(None) for _ in self.shape) + elif not isinstance(selection, tuple): + selection = (selection,) + if any( + part is None or isinstance(part, bool) or not isinstance(part, (slice, int, type(Ellipsis))) + for part in selection + ): + return self._values()[selection] + if sum(part is Ellipsis for part in selection) > 1: + raise IndexError("Only one ellipsis is allowed") + if Ellipsis in selection: + position = selection.index(Ellipsis) + selection = ( + selection[:position] + + (slice(None),) * (len(self.shape) - len(selection) + 1) + + selection[position + 1 :] + ) + selection += (slice(None),) * (len(self.shape) - len(selection)) + if len(selection) != len(self.shape): + raise IndexError("Too many indices") + selected, squeeze = [], [] + for axis, (part, size) in enumerate(zip(selection, self.shape, strict=True)): + if isinstance(part, int): + index = part + size if part < 0 else part + if not 0 <= index < size: + raise IndexError("Portable output index out of range") + selected.append(np.array([index], dtype=np.intp)) + squeeze.append(axis) + else: + selected.append(np.arange(*part.indices(size), dtype=np.intp)) + result = np.empty(tuple(len(indices) for indices in selected), dtype=self.dtype) + scalar = self.kernel.result_cardinality == "block_scalar" + groups = [ + np.unique(indices if scalar else indices // p) + for indices, p in zip(selected, self._partitions, strict=True) + ] + # Always read complete original groups. Output slicing/rechunking never + # changes reduction membership or the domain used by native coordinates. + for index in product(*groups): + index = tuple(int(i) for i in index) + origin = tuple(i * p for i, p in zip(index, self._partitions, strict=True)) + extent = tuple( + min(p, n - o) for p, n, o in zip(self._partitions, self._domain, origin, strict=True) + ) + item = tuple(slice(o, o + n) for o, n in zip(origin, extent, strict=True)) + operands = {} + for name, value in self.inputs.items(): + rank = len(value.shape) + source_item = ( + tuple( + slice(0, 1) if size == 1 else part + for size, part in zip(value.shape, item[len(item) - rank :], strict=True) + ) + if rank + else () + ) + operands[name] = np.broadcast_to(np.asarray(value[source_item]), extent) + context = ( + {"logical_shape": self._domain, "block_origin": origin} if self.kernel.context_ndim else {} + ) + block = self.kernel.evaluate_block(operands, block_shape=extent, **context) + positions = [ + np.flatnonzero(indices == i) + if scalar + else np.flatnonzero((indices >= o) & (indices < o + n)) + for indices, i, o, n in zip(selected, index, origin, extent, strict=True) + ] + local = [indices[pos] - o for indices, pos, o in zip(selected, positions, origin, strict=True)] + if not self.shape: + result[...] = block + else: + result[np.ix_(*positions)] = block if scalar else block[np.ix_(*local)] + return np.squeeze(result, axis=tuple(squeeze)) if squeeze else result + + def __getitem__(self, item): + return self._values(item) + + def compute(self, item=None, **kwargs): + if item is not None: + return blosc2.asarray(self._values(item), **kwargs) + output = blosc2.empty(self.shape, dtype=self.dtype, **kwargs) + if not self.shape: + output[()] = self._values() + return output + # Output storage tiles may intersect several immutable source groups. + # Re-evaluation at a tile boundary is permitted, repartitioning is not. + for origin in product(*(range(0, n, c) for n, c in zip(self.shape, output.chunks, strict=True))): + item = tuple( + slice(o, min(o + c, n)) for o, c, n in zip(origin, output.chunks, self.shape, strict=True) + ) + output[item] = self._values(item) + return output + + def rechunk(self, *, chunks=None, blocks=None, **kwargs): + """Materialize into a different storage grid, retaining logical groups.""" + return self.compute(chunks=chunks, blocks=blocks, **kwargs) + + def sort(self, order=None): + raise NotImplementedError("Portable sorting is not part of the kernel contract") + + def argsort(self, order=None): + raise NotImplementedError("Portable sorting is not part of the kernel contract") + + def save(self, urlpath=None, **kwargs): + from blosc2.b2objects import ( + encode_b2object_payload, + make_b2object_carrier, + write_b2object_payload, + write_b2object_user_vlmeta, + ) + + if urlpath is None: + raise ValueError("A portable carrier requires urlpath") + payload = encode_b2object_payload(self) # Preflight every reference before opening destination. + from blosc2.msgpack_utils import msgpack_packb + + msgpack_packb(payload) + msgpack_packb(self._get_user_vlmeta()) + carrier = make_b2object_carrier("portable", self.shape, self.dtype, urlpath=urlpath, **kwargs) + write_b2object_payload(carrier, payload) + write_b2object_user_vlmeta(carrier, self._get_user_vlmeta()) + self.array, self.schunk = carrier, carrier.schunk + + def to_cframe(self, **kwargs): + from blosc2.b2objects import ( + encode_b2object_payload, + make_b2object_carrier, + write_b2object_payload, + write_b2object_user_vlmeta, + ) + + payload = encode_b2object_payload(self) + carrier = make_b2object_carrier("portable", self.shape, self.dtype, **kwargs) + write_b2object_payload(carrier, payload) + write_b2object_user_vlmeta(carrier, self._get_user_vlmeta()) + return carrier.to_cframe() diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index aa058e044..d53232eba 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1976,7 +1976,9 @@ def process_opened_object(res, *, deserialize=DeserializeMode.SAFE): return proxy if "b2o" in meta: - if deserialize is DeserializeMode.SAFE: + if deserialize is DeserializeMode.SAFE and ( + not isinstance(meta["b2o"], dict) or meta["b2o"].get("kind") not in {"portable", "lazyexpr"} + ): marker = meta["b2o"] kind = marker.get("kind", "b2o") if isinstance(marker, dict) else "b2o" raise UnsafeDeserializationError(str(kind)) diff --git a/src/blosc2/tree_store.py b/src/blosc2/tree_store.py index bdc390390..dc46921ba 100644 --- a/src/blosc2/tree_store.py +++ b/src/blosc2/tree_store.py @@ -441,6 +441,9 @@ def __setitem__( """ key = self._validate_key(key) + from blosc2.b2objects import preflight_persistence + + preflight_persistence(value) if isinstance(value, blosc2.RemoteStore): self._set_remote_store_reference(key, value) return diff --git a/src/blosc2/version.py b/src/blosc2/version.py index 241d027ca..7e3c7039c 100644 --- a/src/blosc2/version.py +++ b/src/blosc2/version.py @@ -1,2 +1,2 @@ -__version__ = "4.14.2.dev0" +__version__ = "4.15.0.dev0" __array_api_version__ = "2024.12" diff --git a/tests/ctable/test_ctable_computed_cols.py b/tests/ctable/test_ctable_computed_cols.py index 9c90040ef..56a13ab47 100644 --- a/tests/ctable/test_ctable_computed_cols.py +++ b/tests/ctable/test_ctable_computed_cols.py @@ -876,16 +876,13 @@ def test_dsl_computed_column_no_storage(): def test_dsl_computed_column_persistence(tmp_path): - """DSL kernel source is persisted and the column is available after open.""" + """Authored computed columns persist with an explicit row-domain contract.""" t = _make_invoice_table(5) - t.add_computed_column("total", total, inputs=["price", "qty"]) + t.add_computed_column("total", total, inputs={"price": "price", "qty": "qty"}, dtype="float64") path = str(tmp_path / "tbl") t.save(path) - - t2 = blosc2.open(path) - assert "total" in t2._computed_cols - assert t2._computed_cols["total"]["kind"] == "dsl" - np.testing.assert_allclose(t2["total"][:], [1.0, 4.0, 9.0, 16.0, 25.0]) + restored = blosc2.CTable.open(path) + np.testing.assert_allclose(restored["total"][:], [1, 4, 9, 16, 25]) def test_dsl_computed_column_inputs_mismatch_raises(): @@ -982,17 +979,14 @@ def test_dsl_generated_column_autofill_append(): def test_dsl_generated_column_persistence_and_append(tmp_path): - """DSL generated column survives save/open and auto-fills after reload.""" + """Authored row transformers persist and refill appended rows natively.""" t = _make_invoice_table(3) - t.add_generated_column("total", values=total, inputs=["price", "qty"]) + t.add_generated_column("total", values=total, inputs={"price": "price", "qty": "qty"}, dtype="float64") path = str(tmp_path / "tbl") t.save(path) - - t2 = blosc2.open(path, mode="a") - assert "total" in t2._materialized_cols - assert t2._materialized_cols["total"]["transformer_kind"] == "dsl" - t2.append((4.0, 4, 0.1)) - np.testing.assert_allclose(t2["total"][:], [1.0, 4.0, 9.0, 16.0]) + restored = blosc2.CTable.open(path, mode="a") + restored.append((4.0, 4, 0.1)) + np.testing.assert_allclose(restored["total"][:], [1.0, 4.0, 9.0, 16.0]) def test_dsl_generated_column_refresh(tmp_path): @@ -1011,8 +1005,8 @@ def test_dsl_generated_column_refresh(tmp_path): @pytest.mark.skipif(sys.platform == "win32", reason="cc backend requires a system C compiler") -def test_dsl_jit_backend_persisted_in_schema(): - """jit_backend set via lazyudf() is stored in the column schema.""" +def test_dsl_jit_backend_not_persisted_in_portable_schema(): + """A local JIT preference does not become a portable execution requirement.""" t = _make_invoice_table(3) t.add_generated_column( "total", @@ -1020,12 +1014,14 @@ def test_dsl_jit_backend_persisted_in_schema(): ) schema = t._schema_dict_with_computed() mat = {m["name"]: m for m in schema["materialized_columns"]} - assert mat["total"]["jit_backend"] == "cc" + assert mat["total"]["transformer_kind"] == "portable" + assert "jit_backend" not in mat["total"] + assert "dsl_source" not in mat["total"] @pytest.mark.skipif(sys.platform == "win32", reason="cc backend requires a system C compiler") -def test_dsl_jit_backend_restored_after_open(tmp_path): - """jit_backend is restored from schema on open and used for auto-fill.""" +def test_dsl_jit_backend_falls_back_after_portable_open(tmp_path): + """A local backend preference allows native portable persistence/fallback.""" t = _make_invoice_table(3) t.add_generated_column( "total", @@ -1033,11 +1029,10 @@ def test_dsl_jit_backend_restored_after_open(tmp_path): ) path = str(tmp_path / "tbl") t.save(path) - - t2 = blosc2.open(path, mode="a") - assert t2._materialized_cols["total"].get("jit_backend") == "cc" - t2.append((4.0, 4, 0.1)) - np.testing.assert_allclose(t2["total"][:], [1.0, 4.0, 9.0, 16.0]) + restored = blosc2.CTable.open(path, mode="a") + restored.append({"price": 4.0, "qty": 4}) + np.testing.assert_allclose(restored["total"][:], [1.0, 4.0, 9.0, 16.0]) + restored.close() def test_dsl_no_jit_backend_not_in_schema(): diff --git a/tests/ctable/test_ctable_dsl_columns.py b/tests/ctable/test_ctable_dsl_columns.py index 6aaf5c2b0..cc40bb363 100644 --- a/tests/ctable/test_ctable_dsl_columns.py +++ b/tests/ctable/test_ctable_dsl_columns.py @@ -86,6 +86,13 @@ def test_dsl_computed_callable_lazyudf_form(): np.testing.assert_array_equal(np.asarray(t["r"][:]), np.arange(20) + 20) +def test_assign_dsl_lazyudf_uses_portable_rows(): + t = _make_table() + view = t.assign(r=blosc2.lazyudf(k_loop, (t.a, t.b), dtype=np.int64)) + assert view._computed_cols["r"]["kind"] == "portable" + np.testing.assert_array_equal(view["r"][:], np.arange(20) + 20) + + def test_dsl_computed_partial_slice(): t = _make_table() t.add_computed_column("r", k_add, inputs=["a", "b"]) @@ -152,23 +159,17 @@ def test_dsl_computed_in_where_streams_multichunk(): def test_dsl_computed_roundtrip(tmp_path): path = str(tmp_path / "dsl.b2d") t = _make_table(urlpath=path, mode="w") - t.add_computed_column("r", k_loop, inputs=["a", "b"]) + t.add_computed_column("r", k_loop, inputs={"x": "a", "y": "b"}) t.close() - # Stored schema carries kind:dsl + dsl_source and no expression. + # New registration stores an artifact rather than Python source. meta = blosc2.open(f"{path}/_meta.b2f") sd = json.loads(meta.vlmeta["schema"]) - (cc,) = sd["computed_columns"] - assert cc["kind"] == "dsl" - assert "dsl_source" in cc - assert "expression" not in cc + assert sd["computed_columns"][0]["kind"] == "portable" + assert "dsl_source" not in sd["computed_columns"][0] t2 = blosc2.open(path) - np.testing.assert_array_equal(np.asarray(t2["r"][:]), np.arange(20) + 20) - a = np.arange(20) - r = a + 20 - sel = t2.where("(r > 25) & (a < 18)")[:] - assert len(sel) == int(((r > 25) & (a < 18)).sum()) + np.testing.assert_array_equal(t2["r"][:], np.arange(20) + 20) t2.close() @@ -230,15 +231,14 @@ def test_dsl_generated_create_index(): def test_dsl_generated_roundtrip(tmp_path): path = str(tmp_path / "gen.b2d") t = _make_table(n=10, urlpath=path, mode="w") - t.add_generated_column("g", values=k_add, inputs=["a", "b"], dtype=blosc2.int64()) + t.add_generated_column("g", values=k_add, inputs={"x": "a", "y": "b"}, dtype=blosc2.int64()) t.close() meta = blosc2.open(f"{path}/_meta.b2f") sd = json.loads(meta.vlmeta["schema"]) - (m,) = sd["materialized_columns"] - assert m["transformer_kind"] == "dsl" - assert "dsl_source" in m + assert sd["materialized_columns"][0]["transformer_kind"] == "portable" + assert "dsl_source" not in sd["materialized_columns"][0] t2 = blosc2.open(path) - np.testing.assert_array_equal(np.asarray(t2["g"][:]), np.arange(10) + 10) + np.testing.assert_array_equal(t2["g"][:], np.arange(10) + 10) t2.close() diff --git a/tests/ndarray/test_dsl_kernels.py b/tests/ndarray/test_dsl_kernels.py index 3d6097fe0..b9747b202 100644 --- a/tests/ndarray/test_dsl_kernels.py +++ b/tests/ndarray/test_dsl_kernels.py @@ -920,17 +920,18 @@ def kernel_save_loop(x, y): return acc -def _save_reload_compute(kernel, inputs_np, inputs_b2, dtype, urlpaths, extra_kwargs=None): - """Save a LazyUDF backed by *kernel*, reload it, and return (reloaded_expr, result).""" +def _save_compute(kernel, inputs_b2, dtype, urlpaths, extra_kwargs=None): + """New authoring persists a native recipe, not a reconstructed Python function.""" lazy = blosc2.lazyudf(kernel, inputs_b2, dtype=dtype, **(extra_kwargs or {})) lazy.save(urlpath=urlpaths["lazy"]) - reloaded = blosc2.open(urlpaths["lazy"], mode="r", deserialize="full") - return reloaded, reloaded.compute() + restored = blosc2.open(urlpaths["lazy"]) + assert restored.partitions == lazy.blocks + assert restored.kernel.schema_version == "1.0" + return restored, restored.compute() def test_dsl_save_simple(tmp_path): - """Simple quadratic kernel: dsl_source and DSLKernel type survive a round-trip.""" - from blosc2.dsl_kernel import DSLKernel + """A quadratic author saves and safely reloads as native portable code.""" shape = (16, 16) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -939,20 +940,15 @@ def test_dsl_save_simple(tmp_path): b = blosc2.asarray(nb, urlpath=str(tmp_path / "b.b2nd"), mode="w") urlpaths = {"lazy": str(tmp_path / "lazy.b2nd")} - reloaded, result = _save_reload_compute(kernel_save_simple, (na, nb), (a, b), np.float64, urlpaths) - - assert isinstance(reloaded, blosc2.LazyUDF) - assert isinstance(reloaded.func, DSLKernel), "func must be a DSLKernel after reload" - assert reloaded.func.dsl_source is not None, "dsl_source must be preserved" - assert "kernel_save_simple" in reloaded.func.dsl_source + reloaded, result = _save_compute(kernel_save_simple, (a, b), np.float64, urlpaths) + assert isinstance(reloaded.kernel, blosc2.PortableKernel) expected = (na + nb) ** 2 # (x+y)^2 == x^2 + y^2 + 2xy np.testing.assert_allclose(result[...], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_clamp(tmp_path): - """Kernel with a `where` call survives save/reload and produces correct values.""" - from blosc2.dsl_kernel import DSLKernel + """Normalized where authoring roundtrips through native persistence.""" shape = (20, 20) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -961,18 +957,15 @@ def test_dsl_save_clamp(tmp_path): b = blosc2.asarray(nb, urlpath=str(tmp_path / "b.b2nd"), mode="w") urlpaths = {"lazy": str(tmp_path / "lazy.b2nd")} - reloaded, result = _save_reload_compute(kernel_save_clamp, (na, nb), (a, b), np.float64, urlpaths) - - assert isinstance(reloaded.func, DSLKernel) - assert reloaded.func.dsl_source is not None + reloaded, result = _save_compute(kernel_save_clamp, (a, b), np.float64, urlpaths) + assert isinstance(reloaded.kernel, blosc2.PortableKernel) expected = np.where(na + nb > 1.5, na + nb, 1.5) np.testing.assert_allclose(result[...], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_loop(tmp_path): - """Kernel with a loop (full DSL function) survives save/reload.""" - from blosc2.dsl_kernel import DSLKernel + """A static loop author roundtrips through native persistence.""" shape = (12, 12) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -981,19 +974,15 @@ def test_dsl_save_loop(tmp_path): b = blosc2.asarray(nb, urlpath=str(tmp_path / "b.b2nd"), mode="w") urlpaths = {"lazy": str(tmp_path / "lazy.b2nd")} - reloaded, result = _save_reload_compute(kernel_save_loop, (na, nb), (a, b), np.float64, urlpaths) - - assert isinstance(reloaded.func, DSLKernel) - assert reloaded.func.dsl_source is not None - assert "for i in range(3):" in reloaded.func.dsl_source + reloaded, result = _save_compute(kernel_save_loop, (a, b), np.float64, urlpaths) + assert isinstance(reloaded.kernel, blosc2.PortableKernel) expected = kernel_save_loop.func(na, nb) np.testing.assert_allclose(result[...], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_getitem(tmp_path): - """Reloaded DSL kernel supports __getitem__ (sliced access), not just compute().""" - from blosc2.dsl_kernel import DSLKernel + """Reloaded native recipes retain sliced access.""" shape = (16, 16) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -1003,16 +992,13 @@ def test_dsl_save_getitem(tmp_path): lazy = blosc2.lazyudf(kernel_save_simple, (a, b), dtype=np.float64) lazy.save(urlpath=str(tmp_path / "lazy.b2nd")) - reloaded = blosc2.open(str(tmp_path / "lazy.b2nd"), mode="r", deserialize="full") - - assert isinstance(reloaded.func, DSLKernel) + reloaded = blosc2.open(tmp_path / "lazy.b2nd") expected = (na + nb) ** 2 np.testing.assert_allclose(reloaded[()], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_input_names_match(tmp_path): - """After reload, input_names in the DSLKernel match the original kernel.""" - from blosc2.dsl_kernel import DSLKernel + """Native recipes preserve named positional bindings.""" shape = (10, 10) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -1022,11 +1008,20 @@ def test_dsl_save_input_names_match(tmp_path): lazy = blosc2.lazyudf(kernel_save_simple, (a, b), dtype=np.float64) lazy.save(urlpath=str(tmp_path / "lazy.b2nd")) - reloaded = blosc2.open(str(tmp_path / "lazy.b2nd"), mode="r", deserialize="full") - - assert isinstance(reloaded.func, DSLKernel) - assert reloaded.func.input_names == ["x", "y"] - assert list(reloaded.inputs_dict.keys()) == ["x", "y"] + reloaded = blosc2.open(tmp_path / "lazy.b2nd") + assert list(reloaded.kernel.input_dtypes) == ["x", "y"] + assert list(reloaded.inputs) == ["x", "y"] + grouped = blosc2.asarray( + np.arange(8, dtype="int64"), chunks=(8,), blocks=(2,), urlpath=tmp_path / "grouped.b2nd", mode="w" + ) + kernel = blosc2.DSLKernel.from_source("def grouped(x):\n return x + sum(x) + _i0\n") + authored = blosc2.lazyudf(kernel, (grouped,), dtype="int64") + frame = authored.to_cframe() + restored = blosc2.from_cframe(frame) + expected = np.arange(8) * 2 + np.repeat(np.arange(8).reshape(4, 2).sum(axis=1), 2) + assert restored.partitions == (2,) + np.testing.assert_array_equal(restored[:], expected) + np.testing.assert_array_equal(restored[1:7:2], expected[1:7:2]) def test_dsl_save_dictstore_operands(tmp_path): @@ -1047,16 +1042,7 @@ def test_dsl_save_dictstore_operands(tmp_path): b = dstore["/b"] lazy = blosc2.lazyudf(kernel_save_simple, (a, b), dtype=np.float64) lazy.save(urlpath=str(expr_path)) - - carrier = blosc2.open(str(expr_path), mode="r", deserialize="full").array - assert carrier.schunk.vlmeta["b2o"]["operands"] == { - "o0": {"kind": "dictstore_key", "version": 1, "urlpath": "ops.b2z", "key": "/a"}, - "o1": {"kind": "dictstore_key", "version": 1, "urlpath": "ops.b2z", "key": "/b"}, - } - - reloaded = blosc2.open(str(expr_path), mode="r", deserialize="full") - expected = (np.arange(shape[0], dtype=np.float64) * 3) ** 2 - np.testing.assert_allclose(reloaded.compute()[...], expected, rtol=1e-5, atol=1e-6) + np.testing.assert_array_equal(blosc2.open(expr_path)[:], (np.arange(10) * 3) ** 2) # --- plans/dsl-glitches.md regressions: G1 (one-per-line), G2 (input reassign), diff --git a/tests/ndarray/test_jit_options.py b/tests/ndarray/test_jit_options.py index dda096266..afb00c643 100644 --- a/tests/ndarray/test_jit_options.py +++ b/tests/ndarray/test_jit_options.py @@ -353,6 +353,53 @@ def test_environment_overrides_and_tcc_ignores_cc_settings(tmp_path, capfd, monk assert capfd.readouterr() == ("", "") +@pytest.mark.parametrize("backend", ["tcc", "cc"]) +def test_synthesized_backend_respects_environment_and_source(monkeypatch, backend): + from blosc2.lazyexpr import _apply_jit_backend_pragma + + monkeypatch.setenv("ME_DSL_JIT_COMPILER", backend) + other = "cc" if backend == "tcc" else "tcc" + inputs = {"x": np.arange(4, dtype=np.float64)} + assert _apply_jit_backend_pragma("x + 1", inputs, other).startswith(f"# me:compiler={backend}\n") + source = f"# me:compiler={other}\ndef k(x):\n return x + 1\n" + assert _apply_jit_backend_pragma(source, inputs, backend) == source + commented = "# me:fp=strict\ndef k(x):\n return x\n" + assert _apply_jit_backend_pragma(commented, inputs, other) == f"# me:compiler={backend}\n{commented}" + + +@pytest.mark.parametrize("backend", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +def test_reduction_backend_preserves_scalar_accumulator(monkeypatch, backend, jit): + import importlib + + lazyexpr = importlib.import_module("blosc2.lazyexpr") + + def reject_elementwise_wrapper(*args): + pytest.fail("Scalar reductions must not be wrapped in elementwise DSL source") + + monkeypatch.setattr(lazyexpr, "_apply_jit_backend_pragma", reject_elementwise_wrapper) + original = blosc2.NDArray._set_pref_expr + calls = [] + + def capture(self, expression, inputs, fp_accuracy, aux_reduc=None, **kwargs): + assert aux_reduc is not None + assert kwargs["jit"] is False # Prevent native auto-lifting as well. + calls.append(expression) + return original(self, expression, inputs, fp_accuracy, aux_reduc, **kwargs) + + monkeypatch.setattr(blosc2.NDArray, "_set_pref_expr", capture) + values = np.arange(128, dtype=np.float64) + expr = blosc2.asarray(values) * 2 + 1 + assert expr.sum(jit=jit, jit_backend=backend) == (values * 2 + 1).sum() + if blosc2.IS_WASM: + # WASM currently uses the non-prefilter reduction path. It must still + # produce correct results without introducing an elementwise wrapper. + assert not calls + else: + assert calls + assert all(expression.startswith("sum(") for expression in calls) + + @pytest.mark.skipif(blosc2.IS_WASM or os.name == "nt", reason="POSIX compiler-output injection") def test_compiler_output(tmp_path, capfd): compiler = tmp_path / "bad cc's executable" diff --git a/tests/ndarray/test_lazyexpr.py b/tests/ndarray/test_lazyexpr.py index 5a099cbac..bb74a1eb7 100644 --- a/tests/ndarray/test_lazyexpr.py +++ b/tests/ndarray/test_lazyexpr.py @@ -487,7 +487,7 @@ def test_arctan2_pow(urlpath, shape_fixture, dtype_fixture, function, value1, va expr = blosc2.LazyExpr(new_op=(a1, function, a2)) if urlpath is not None: expr.save(urlpath=urlpath_save) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr if function == "**": @@ -501,7 +501,7 @@ def test_arctan2_pow(urlpath, shape_fixture, dtype_fixture, function, value1, va expr = blosc2.LazyExpr(new_op=(a1, function, value2)) if urlpath is not None: expr.save(urlpath=urlpath_save) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr if function == "**": @@ -517,7 +517,7 @@ def test_arctan2_pow(urlpath, shape_fixture, dtype_fixture, function, value1, va expr = blosc2.LazyExpr(new_op=(value1, function, a2)) if urlpath is not None: expr.save(urlpath=urlpath_save) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr if function == "**": @@ -737,7 +737,7 @@ def test_save(): ) np.testing.assert_allclose(res[:], nres, rtol=tol, atol=tol) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") # After opening, check that a lazy expression does have an array # and schunk attributes. This is to allow the .info() method to work. assert hasattr(expr, "array") is True @@ -756,7 +756,7 @@ def test_save(): var_dict = {"a1": ops[0], "a2": ops[1], "a3": ops[2], "a4": ops[3], "x": x} lazy_expr = eval(expr, var_dict) lazy_expr.save(urlpath=urlpath_save2) - expr = blosc2.open(urlpath_save2, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save2, mode="r") assert expr.array.dtype == np.float64 res = expr.compute() nres = ne_evaluate("na1 / na2 + na2 - na3 * na4**3") @@ -780,7 +780,7 @@ def test_save_unsafe(): expr.save(urlpath=urlpath) disk_arrays.append(urlpath) - expr = blosc2.open(urlpath, mode="r", deserialize="full") + expr = blosc2.open(urlpath, mode="r") # Replace expression by a (potentially) unsafe expression expr.expression = "import os; os.system('touch /tmp/unsafe')" with pytest.raises(ValueError) as excinfo: @@ -828,7 +828,7 @@ def test_save_functions(function, dtype_fixture, shape_fixture): expr = blosc2.LazyExpr(new_op=(a1, function, None)) expr.save(urlpath=urlpath_save) del expr - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr @@ -844,7 +844,7 @@ def test_save_functions(function, dtype_fixture, shape_fixture): res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol) @@ -868,7 +868,7 @@ def test_save_contains(values): # Construct the lazy expression expr_lazy = blosc2.LazyExpr(new_op=(a1_blosc, "contains", value2)) expr_lazy.save(urlpath=urlpath_save) - expr_lazy = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr_lazy = blosc2.open(urlpath_save, mode="r") # Evaluate using NumExpr expr_numexpr = f"{'contains'}(a1, value2)" res_numexpr = ne_evaluate(expr_numexpr) @@ -878,7 +878,7 @@ def test_save_contains(values): # Construct the lazy expression expr_lazy = blosc2.LazyExpr(new_op=(a1_blosc, "contains", a2_blosc)) expr_lazy.save(urlpath=urlpath_save) - expr_lazy = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr_lazy = blosc2.open(urlpath_save, mode="r") # Evaluate using NumExpr res_numexpr = ne_evaluate("contains(a2, a1)") else: # ("str", "NDArray") @@ -888,7 +888,7 @@ def test_save_contains(values): # Construct the lazy expression expr_lazy = blosc2.LazyExpr(new_op=(value1, "contains", a2_blosc)) expr_lazy.save(urlpath=urlpath_save) - expr_lazy = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr_lazy = blosc2.open(urlpath_save, mode="r") # Evaluate using NumExpr res_numexpr = ne_evaluate("contains(value1, a2)") res_lazyexpr = expr_lazy.compute() @@ -922,7 +922,7 @@ def test_save_many_functions(dtype_fixture, shape_fixture): res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol, atol=atol) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol, atol=atol) @@ -976,7 +976,7 @@ def test_save_constructor(disk, shape, dtype, constructor): a = b2func(lshape, dtype=dtype, shape=shape, urlpath=urlpath, mode="w") expr = f"a + {constructor}({lshape}, dtype={dtype}, shape={shape}) + 1" if disk: - a = blosc2.open(urlpath, mode="r", deserialize="full") + a = blosc2.open(urlpath, mode="r") npfunc = getattr(np, constructor) if constructor == "linspace": na = npfunc(0, 10, lshape, dtype=dtype).reshape(shape) @@ -994,7 +994,7 @@ def test_save_constructor(disk, shape, dtype, constructor): assert lexpr.shape == a.shape if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() nres = na + na + 1 assert np.allclose(res[()], nres) @@ -1016,7 +1016,7 @@ def test_save_2_constructors(shape, disk): lexpr = blosc2.lazyexpr(expr) if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() na = np.arange(lshape).reshape(shape) nb = np.ones(shape) @@ -1043,7 +1043,7 @@ def test_save_constructor_reshape(shape, disk): lexpr = blosc2.lazyexpr(expr) if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() na = np.arange(lshape).reshape(shape) nb = np.ones(shape) @@ -1067,7 +1067,7 @@ def test_save_2equal_constructors(shape, disk): lexpr = blosc2.lazyexpr(expr) if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() na = np.ones(shape, dtype=np.int8) nb = np.ones(shape) @@ -1374,9 +1374,9 @@ def test_fill_disk_operands(chunks, blocks, disk, fill_value): b = blosc2.zeros((N, N), urlpath=bpath, mode="w", chunks=chunks, blocks=blocks) c = blosc2.zeros((N, N), urlpath=cpath, mode="w", chunks=chunks, blocks=blocks) if disk: - a = blosc2.open("a.b2nd", mode="r", deserialize="full") - b = blosc2.open("b.b2nd", mode="r", deserialize="full") - c = blosc2.open("c.b2nd", mode="r", deserialize="full") + a = blosc2.open("a.b2nd", mode="r") + b = blosc2.open("b.b2nd", mode="r") + c = blosc2.open("c.b2nd", mode="r") expr = ((a**3 + blosc2.sin(c * 2)) < b) & ~(c > 0) @@ -1720,7 +1720,7 @@ def test_missing_operator(): blosc2.remove_urlpath("b.b2nd") # Re-open the lazy expression with pytest.raises(blosc2.exceptions.MissingOperands) as excinfo: - blosc2.open("expr.b2nd", mode="r", deserialize="full") + blosc2.open("expr.b2nd", mode="r") # Check that some operand is missing assert "a" not in excinfo.value.missing_ops @@ -1751,13 +1751,13 @@ def test_save_dictstore_operands(tmp_path): expr = blosc2.lazyexpr("a + b") expr.save(expr_path) - carrier = blosc2.open(expr_path, mode="r", deserialize="full").array + carrier = blosc2.open(expr_path, mode="r").array assert carrier.schunk.vlmeta["b2o"]["operands"] == { "a": {"kind": "dictstore_key", "version": 1, "urlpath": str(store_path), "key": "/a"}, "b": {"kind": "dictstore_key", "version": 1, "urlpath": str(store_path), "key": "/b"}, } - restored = blosc2.open(expr_path, mode="r", deserialize="full") + restored = blosc2.open(expr_path, mode="r") assert isinstance(restored, blosc2.LazyExpr) np.testing.assert_array_equal(restored[:], expected) @@ -1773,6 +1773,9 @@ def test_save_proxy_operands_reopen_default_mode(tmp_path): expr = proxy + proxy expr.save(str(expr_path)) + # Proxy reconstruction remains full-only, even inside an ordinary expression. + with pytest.raises(blosc2.UnsafeDeserializationError, match="proxy"): + blosc2.open(str(expr_path), mode="r") restored = blosc2.open(str(expr_path), mode="r", deserialize="full") assert isinstance(restored, blosc2.LazyExpr) @@ -1795,7 +1798,7 @@ def test_lazyexpr_vlmeta_in_memory_and_persisted(tmp_path): expr_path = tmp_path / "expr_vlmeta.b2nd" expr.save(str(expr_path)) - restored = blosc2.open(str(expr_path), mode="r", deserialize="full") + restored = blosc2.open(str(expr_path), mode="r") assert restored.vlmeta["name"] == "sum" assert restored.vlmeta["config"] == {"scale": 1} @@ -1803,9 +1806,9 @@ def test_lazyexpr_vlmeta_in_memory_and_persisted(tmp_path): with pytest.raises(ValueError, match="reading mode"): restored.vlmeta["note"] = "persisted" - writable = blosc2.open(str(expr_path), mode="a", deserialize="full") + writable = blosc2.open(str(expr_path), mode="a") writable.vlmeta["note"] = "persisted" - reopened = blosc2.open(str(expr_path), mode="r", deserialize="full") + reopened = blosc2.open(str(expr_path), mode="r") assert reopened.vlmeta["note"] == "persisted" np.testing.assert_array_equal(reopened[:], np.arange(5, dtype=np.int64) * 2) @@ -1881,19 +1884,19 @@ def test_chain_persistentexpressions(): le1_ = blosc2.lazyexpr("a ** 3 + sin(a ** 2)", {"a": a}) le1_.save("expr1.b2nd", mode="w") - myle1 = blosc2.open("expr1.b2nd", mode="r", deserialize="full") + myle1 = blosc2.open("expr1.b2nd", mode="r") le2_ = blosc2.lazyexpr("(le1 < c)", {"le1": myle1, "c": c}) le2_.save("expr2.b2nd", mode="w") - myle2 = blosc2.open("expr2.b2nd", mode="r", deserialize="full") + myle2 = blosc2.open("expr2.b2nd", mode="r") le3_ = blosc2.lazyexpr("(b < 0)", {"b": b}) le3_.save("expr3.b2nd", mode="w") - myle3 = blosc2.open("expr3.b2nd", mode="r", deserialize="full") + myle3 = blosc2.open("expr3.b2nd", mode="r") le4_ = blosc2.lazyexpr("(le2 & le3)", {"le2": myle2, "le3": myle3}) le4_.save("expr4.b2nd", mode="w") - myle4 = blosc2.open("expr4.b2nd", mode="r", deserialize="full") + myle4 = blosc2.open("expr4.b2nd", mode="r") assert (myle4[:] == le4[:]).all() # Remove files diff --git a/tests/ndarray/test_lazyudf.py b/tests/ndarray/test_lazyudf.py index b75f292d0..39811f9ad 100644 --- a/tests/ndarray/test_lazyudf.py +++ b/tests/ndarray/test_lazyudf.py @@ -483,9 +483,8 @@ def test_save_ludf(): expr = blosc2.lazyudf(udf1p, (array,), np.float64, cparams=cparams) - expr.save(urlpath=urlpath) - del expr - expr = blosc2.open(urlpath, mode="r", deserialize="full") + with pytest.raises(TypeError, match="PortableKernel"): + expr.save(urlpath=urlpath) assert isinstance(expr, blosc2.LazyUDF) res_lazyexpr = expr.compute() np.testing.assert_array_equal(res_lazyexpr[:], npc) @@ -493,9 +492,8 @@ def test_save_ludf(): if blosc2._HAS_NUMBA: expr = blosc2.lazyudf(udf1p_numba, (array,), np.float64) - expr.save(urlpath=urlpath) - del expr - expr = blosc2.open(urlpath, mode="r", deserialize="full") + with pytest.raises(TypeError, match="PortableKernel"): + expr.save(urlpath=urlpath) assert isinstance(expr, blosc2.LazyUDF) res_lazyexpr = expr.compute() np.testing.assert_array_equal(res_lazyexpr[:], npc) @@ -511,18 +509,11 @@ def test_lazyudf_vlmeta_roundtrip(tmp_path): expr.vlmeta["name"] = "increment" expr.vlmeta["attrs"] = {"version": 1} - expr.save(urlpath=str(expr_path)) - - restored = blosc2.open(str(expr_path), mode="r", deserialize="full") - - assert isinstance(restored, blosc2.LazyUDF) - assert restored.vlmeta["name"] == "increment" - assert restored.vlmeta["attrs"] == {"version": 1} - - with blosc2.open(str(expr_path), mode="r", deserialize="full") as restored_ctx: - assert isinstance(restored_ctx, blosc2.LazyUDF) - assert restored_ctx.vlmeta["name"] == "increment" - assert restored_ctx.vlmeta["attrs"] == {"version": 1} + with pytest.raises(TypeError, match="PortableKernel"): + expr.save(urlpath=str(expr_path)) + assert not expr_path.exists() + assert expr.vlmeta["name"] == "increment" + assert expr.vlmeta["attrs"] == {"version": 1} # Test get_chunk method diff --git a/tests/test_b2objects.py b/tests/test_b2objects.py index 9eb0cde16..921d5246a 100644 --- a/tests/test_b2objects.py +++ b/tests/test_b2objects.py @@ -10,6 +10,7 @@ from pathlib import Path import numpy as np +import pytest import blosc2 import blosc2.c2array as blosc2_c2array @@ -148,27 +149,25 @@ def test_lazyudf_from_cframe_roundtrip(tmp_path): a = blosc2.asarray(np.arange(5, dtype=np.float64), urlpath=tmp_path / "a.b2nd", mode="w") b = blosc2.asarray(np.arange(5, dtype=np.float64) * 2, urlpath=tmp_path / "b.b2nd", mode="w") expr = blosc2.lazyudf(kernel_add_square, (a, b), dtype=np.float64) - carrier = blosc2.ndarray_from_cframe(expr.to_cframe()) + restored = blosc2.from_cframe(expr.to_cframe()) + np.testing.assert_allclose(restored[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + from blosc2.msgpack_utils import msgpack_packb, msgpack_unpackb - assert carrier.schunk.meta["b2o"] == {"kind": "lazyudf", "version": 1} - payload = carrier.schunk.vlmeta["b2o"] - assert payload["kind"] == "lazyudf" - assert payload["version"] == 1 - assert payload["function_kind"] == "dsl" - assert payload["dsl_version"] == 1 - assert payload["name"] == "kernel_add_square" - assert "kernel_add_square" in payload["udf_source"] - assert payload["dtype"] == np.dtype(np.float64).str - assert payload["shape"] == [5] - assert payload["operands"] == { - "o0": {"kind": "urlpath", "version": 1, "urlpath": (tmp_path / "a.b2nd").as_posix()}, - "o1": {"kind": "urlpath", "version": 1, "urlpath": (tmp_path / "b.b2nd").as_posix()}, - } + offset = np.float32(2) - restored = blosc2.from_cframe(expr.to_cframe(), deserialize="full") + @blosc2.dsl_kernel + def captured(x, scale): + return x * scale + offset - assert isinstance(restored, blosc2.LazyUDF) - np.testing.assert_allclose(restored[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + authored = blosc2.lazyudf(captured, (a, np.int16(3)), dtype="float64") + authored.vlmeta["label"] = "captured" + packed = msgpack_packb([authored]) + framed = authored.to_cframe() + offset = np.float32(99) + np.testing.assert_array_equal(msgpack_unpackb(packed)[0][:], np.arange(5) * 3 + 2) + framed_recipe = blosc2.from_cframe(framed) + np.testing.assert_array_equal(framed_recipe[:], np.arange(5) * 3 + 2) + assert framed_recipe.vlmeta["label"] == "captured" def test_lazyudf_open_roundtrip(tmp_path): @@ -178,10 +177,27 @@ def test_lazyudf_open_roundtrip(tmp_path): urlpath = tmp_path / "expr.b2nd" expr.save(urlpath) - restored = blosc2.open(urlpath, mode="r", deserialize="full") - - assert isinstance(restored, blosc2.LazyUDF) + restored = blosc2.open(urlpath, mode="r") np.testing.assert_allclose(restored[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + nested = expr + 1 + nested.save(tmp_path / "nested.b2nd") + np.testing.assert_array_equal(blosc2.open(tmp_path / "nested.b2nd")[:], (np.arange(5) * 3) ** 2 + 1) + for body, expected in ( + ("print(x)\n return x", "print"), + ("return callback(x)", "invalid_source"), + ("return sum(x)", "block scalar"), + ): + invalid = blosc2.DSLKernel.from_source(f"def invalid(x):\n {body}\n") + bad = blosc2.lazyudf(invalid, (a,), dtype="float64") + before = urlpath.read_bytes() + with pytest.raises((blosc2.PortableArtifactError, ValueError), match=expected): + bad.save(urlpath, mode="w") + assert urlpath.read_bytes() == before + fixture = Path(__file__).parent / "data" / "legacy_lazyudf_v1" / "expr.b2nd" + legacy = blosc2.open(fixture, deserialize="full") + with pytest.raises(TypeError, match="legacy"): + legacy.save(urlpath, mode="w") + assert urlpath.read_bytes() == before def test_b2z_bundle_with_lazy_recipes_opens_read_only(tmp_path): @@ -197,13 +213,11 @@ def test_b2z_bundle_with_lazy_recipes_opens_read_only(tmp_path): udf = blosc2.lazyudf(kernel_add_square, (a, b), dtype=np.float64, shape=a.shape) store["/recipes/expr"] = blosc2.ndarray_from_cframe(expr.to_cframe()) - store["/recipes/udf"] = blosc2.ndarray_from_cframe(udf.to_cframe()) + store["/recipes/udf"] = udf with blosc2.open(str(bundle_path), mode="r", deserialize="full") as store: restored_expr = store["/recipes/expr"] - restored_udf = store["/recipes/udf"] assert isinstance(restored_expr, blosc2.LazyExpr) - assert isinstance(restored_udf, blosc2.LazyUDF) np.testing.assert_allclose(restored_expr.compute()[:], np.arange(5, dtype=np.float64) * 3) - np.testing.assert_allclose(restored_udf.compute()[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + np.testing.assert_allclose(store["/recipes/udf"][:], (np.arange(5, dtype=np.float64) * 3) ** 2) diff --git a/tests/test_deserialization.py b/tests/test_deserialization.py index 1aae429dd..8e3cba9c8 100644 --- a/tests/test_deserialization.py +++ b/tests/test_deserialization.py @@ -180,13 +180,23 @@ def test_active_top_level_object_requires_full(tmp_path): path = tmp_path / "expression.b2nd" expression.save(path) - with pytest.raises(blosc2.UnsafeDeserializationError, match="lazyexpr"): - blosc2.open(path) + np.testing.assert_array_equal(blosc2.open(path)[:], np.arange(4) + 1) trusted = blosc2.open(path, deserialize="full") np.testing.assert_array_equal(trusted[:], np.arange(4) + 1) +@pytest.mark.parametrize("expression", ["sqrt(x)", "sum(x)", "x.mean()", "x[0]"]) +def test_ordinary_lazyexpr_roundtrip_defaults_safe(tmp_path, expression): + values = np.arange(1, 5, dtype="float64") + operand = blosc2.asarray(values, urlpath=tmp_path / "operand.b2nd", mode="w") + lazy = blosc2.lazyexpr(expression, operands={"x": operand}) + expected = lazy[()] + path = tmp_path / "expression.b2nd" + lazy.save(path) + np.testing.assert_allclose(blosc2.open(path)[()], expected) + + @pytest.mark.parametrize("value", [True, False, "unknown", None]) def test_invalid_deserialize_mode(value): source = blosc2.SChunk() diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py new file mode 100644 index 000000000..8db23ddf7 --- /dev/null +++ b/tests/test_dsl_portable.py @@ -0,0 +1,270 @@ +"""Representative Python host conformance; exhaustive fixtures belong to miniexpr.""" + +import os +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +import blosc2 +from blosc2.dsl_kernel import DSLKernel + +# External conformance is opt-in; never infer an unrelated sibling checkout. +# This may point at tests/portable-dsl in CMake's pinned miniexpr fetch. +CORPUS = Path(os.environ["MINIEXPR_PORTABLE_CORPUS"]) if os.environ.get("MINIEXPR_PORTABLE_CORPUS") else None + + +# Native CTest runs every fixture under its supported execution engines. Python +# tests only distinct host boundaries here and representative output conversions; +# numerical/JIT scenarios are covered separately by test_portable_artifact.py. +CASES = [ + "affine", + "special_floats", + "math", + "division_bool", + "division_locals", + "while_cap_exceeded", + "unknown_name", + "zero_step", + "missing_return_mixed", + "missing_return_loop", + "missing_return_loop_ok", +] +TYPES = ("bool", "int32", "int64", "float32", "float64") +CASES += [ + f"convert_{input_type}_{output_type}" + for input_type, output_type in ( + ("bool", "int64"), + ("int32", "bool"), + ("int64", "float32"), + ("float32", "float64"), + ("float64", "int32"), + ) +] + + +def corpus_source(case): + return CORPUS / ("identity.dsl" if case.startswith("convert_") else f"{case}.dsl") + + +def typed_values(values, dtype): + # NumPy treats nonempty strings as True; fixture Boolean tokens are 0/1. + return np.array([int(value) for value in values] if dtype == np.dtype("bool") else values, dtype=dtype) + + +def assert_typed_result(actual, expected, *, exact=False): + assert actual.dtype == expected.dtype + if expected.dtype.kind in "biu" or exact: + np.testing.assert_array_equal(actual, expected) + if expected.dtype.kind == "f": + if not exact: + tolerance = 1e-6 if expected.dtype == np.dtype("float32") else 1e-12 + np.testing.assert_allclose(actual, expected, rtol=tolerance, atol=tolerance, equal_nan=True) + zeros = expected == 0 + np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) + + +@pytest.mark.parametrize("case", CASES) +def test_native_source_corpus(case, portable_validator, monkeypatch): + if case.startswith("while_cap_"): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + if CORPUS is None: + pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") + # An explicit corpus must match draft 1.0; never silently skip missing files. + source = corpus_source(case).read_text() + words = (CORPUS / f"{case}.txt").read_text().split() + outcome, input_type, output_type = words[:3] + count, nvars = map(int, words[3:5]) + names = words[5 : 5 + nvars] + info = portable_validator(source, dict.fromkeys(names, input_type), output_type) + assert info["valid"] == (outcome != "compile_error"), info + rows = np.array(words[5 + nvars :]).reshape(count, nvars + 1) + input_dtype, output_dtype = np.dtype(input_type), np.dtype(output_type) + expected = typed_values(rows[:, -1], output_dtype) + kernel = DSLKernel.from_source(source) + by_name = {name: typed_values(rows[:, index], input_dtype) for index, name in enumerate(names)} + if outcome in {"ok", "ok_exact"}: + artifact = kernel.export(dict.fromkeys(names, input_type), output_type) + loaded = blosc2.PortableKernel.from_json(artifact, jit=True) + assert not loaded.has_jit + result = loaded.evaluate(by_name) + assert_typed_result(result, expected, exact=outcome == "ok_exact") + elif outcome == "compile_error": + with pytest.raises(blosc2.PortableArtifactError): + kernel.export(dict.fromkeys(names, input_type), output_type) + else: + artifact = kernel.export(dict.fromkeys(names, input_type), output_type) + loaded = blosc2.PortableKernel.from_json(artifact, jit=True) + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + loaded.evaluate(by_name) + assert kernel.dsl_source == source + assert kernel.func is None + + runner = os.environ.get("MINIEXPR_PORTABLE_RUNNER") + if runner: + completed = subprocess.run( + [ + runner, + str(corpus_source(case)), + str(CORPUS / f"{case}.txt"), + "on", + ], + check=True, + capture_output=True, + text=True, + ) + lines = completed.stdout.splitlines() + if outcome == "compile_error": + assert lines == [outcome] + else: + assert lines[0] == "jit=0" + if outcome == "eval_error": + assert lines[1:] == [outcome] + else: + assert_typed_result( + typed_values(lines[1:], output_dtype), expected, exact=outcome == "ok_exact" + ) + + +def test_from_source_never_executes_python(tmp_path): + target = tmp_path / "executed" + with pytest.raises(ValueError, match="start with a function"): + DSLKernel.from_source(f"open({str(target)!r}, 'w')\ndef kernel(x):\n return x\n") + with pytest.raises(ValueError, match="plain positional"): + DSLKernel.from_source(f"def kernel(x=open({str(target)!r}, 'w')):\n return x\n") + assert not target.exists() + + +@pytest.mark.parametrize( + "source", ["", "def broken(", "def k(x, x):\n return x", "def k(x):\n return x\x00"] +) +def test_from_source_invalid_header(source): + with pytest.raises(ValueError): + DSLKernel.from_source(source) + + +def test_from_source_native_body_and_no_python_fallback(): + source = "# me:compiler=tcc\ndef kernel(x):\n return !x\n" + kernel = DSLKernel.from_source(source) + assert kernel.dsl_source == source + assert kernel.input_names == ["x"] + with pytest.raises(RuntimeError, match="cannot execute as a Python"): + kernel((np.array([1.0]),), np.empty(1)) + + +@pytest.mark.parametrize("jit", [False, True]) +def test_from_source_without_external_corpus(jit): + source = "# me:compiler=tcc\ndef kernel(x):\n return !x\n" + kernel = blosc2.DSLKernel.from_source(source) + result = blosc2.lazyudf(kernel, (np.array([0.0, 1.0, -1.0]),), dtype=np.bool_, jit=jit)[:] + np.testing.assert_array_equal(result, [True, False, False]) + + +def test_from_source_invalid_native_body(): + kernel = DSLKernel.from_source("def kernel(x):\n return x[0]\n") + with pytest.raises((NotImplementedError, RuntimeError)): + blosc2.lazyudf(kernel, (np.arange(4, dtype=np.float64),), dtype=np.float64)[:] + + +@pytest.fixture +def portable_validator(): + info = blosc2.validate_portable_dsl("def k(x):\n return x\n", {"x": "float64"}, "float64") + if info["status"] == "runtime_unsupported": + pytest.skip("Rebuild with miniexpr portable validation support") + assert info == {"valid": True, "status": "success", "line": 0, "column": 0, "error": None} + return blosc2.validate_portable_dsl + + +@pytest.mark.parametrize( + ("body", "status"), + [ + ("print(x); return x", "invalid_source"), + ("return np.sin(x)", "invalid_source"), + ("return x.lower()", "invalid_source"), + ("return x[0]", "invalid_source"), + ("return _flat_idx + x", "invalid_source"), + ("return callback(x)", "invalid_source"), + ("return CAPTURE + x", "invalid_source"), + ("return sin()", "invalid_source"), + ], +) +def test_portable_profile_rejections(portable_validator, body, status): + info = portable_validator(f"def k(x):\n {body}\n", {"x": "float64"}, "float64") + assert not info["valid"] + assert info["status"] == status + assert info["error"] + if "_flat_idx" in body: + # Descriptor-wide missing-context diagnostic is not a source location. + assert info["line"] == 0 + else: + assert info["line"] == 2 + assert info["column"] > 0 + + +def test_portable_profile_signature_and_version(portable_validator): + source = "def k(x, y):\n return x + y\n" + inputs = {"y": "float64", "x": "float64"} + assert portable_validator(source, inputs, "float64")["valid"] + assert portable_validator(source, inputs, "float64", language_version="99")["status"] == ( + "unsupported_version" + ) + assert portable_validator(source, {"x": "float64"}, "float64")["status"] == "invalid_signature" + assert portable_validator(source, {"x": "float64", "y": "int32"}, "float64")["valid"] + assert ( + portable_validator(source, inputs, "float64", language_version="0.1")["status"] + == "unsupported_version" + ) + assert portable_validator(source, inputs, "complex128")["status"] == "unsupported_feature" + + +def test_portable_profile_never_executes_or_compiles_jit(portable_validator, monkeypatch, tmp_path): + cache = tmp_path / "jit-cache" + target = tmp_path / "executed" + monkeypatch.setenv("CC", "/no/compiler") + monkeypatch.setenv("ME_DSL_JIT_COMPILER", "cc") + monkeypatch.setenv("ME_DSL_JIT", "1") + monkeypatch.setenv("ME_DSL_FP_MODE", "fast") + monkeypatch.setenv("ME_DSL_JIT_CACHE_DIR", str(cache)) + # This would reach the iteration cap if executed. Validation must still succeed. + source = "# me:compiler=cc\ndef k(x):\n while 1:\n pass\n return x\n" + assert portable_validator(source, {"x": "float64"}, "float64")["valid"] + source = f"def k(x):\n return open({str(target)!r}, 'w')\n" + assert not portable_validator(source, {"x": "float64"}, "float64")["valid"] + assert not target.exists() + assert not cache.exists() + + +def test_portable_profile_missing_native_support(monkeypatch): + from blosc2 import blosc2_ext + + monkeypatch.delattr(blosc2_ext, "validate_portable_dsl_source", raising=False) + info = blosc2.validate_portable_dsl("def k(x):\n return x\n", {"x": "float64"}, "float64") + assert not info["valid"] + assert info["status"] == "runtime_unsupported" + + +@pytest.mark.parametrize( + ("source", "inputs", "version"), + [ + ("def k(x):\n return x\n\x00", {"x": "float64"}, "1.0"), + ("def k(x):\n return x\n", {"x\x00": "float64"}, "1.0"), + ("def k(x):\n return x\n", {"x": "float64"}, "1.0\x00"), + ], +) +def test_portable_profile_rejects_nul(source, inputs, version): + with pytest.raises(ValueError, match="NUL"): + blosc2.validate_portable_dsl(source, inputs, "float64", language_version=version) + + +@pytest.mark.parametrize("literal", ["9007199254740993", "9_007_199_254_740_993", "0x20000000000001"]) +def test_portable_integer_literal_bound_is_exact(portable_validator, literal): + info = portable_validator(f"def k(x):\n return x + {literal}\n", {"x": "int64"}, "int64") + assert info["valid"], info + author = DSLKernel.from_source(f"def k(x):\n return x + {literal}\n") + kernel = blosc2.PortableKernel.from_json(author.export({"x": "int64"}, "int64")) + np.testing.assert_array_equal( + kernel.evaluate({"x": np.array([0, 1], dtype="int64")}), [2**53 + 1, 2**53 + 2] + ) + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + kernel.evaluate({"x": np.array([np.iinfo("int64").max], dtype="int64")}) diff --git a/tests/test_objectarray.py b/tests/test_objectarray.py index 53c526c59..616746e80 100644 --- a/tests/test_objectarray.py +++ b/tests/test_objectarray.py @@ -267,9 +267,8 @@ def test_msgpack_supports_lazyudf_dslkernel(tmp_path): oarr = blosc2.ObjectArray() oarr.append(udf) restored = oarr[0] - - assert isinstance(restored, blosc2.LazyUDF) - np.testing.assert_allclose(restored[:], expected) + np.testing.assert_array_equal(restored[:], expected) + assert restored.kernel.schema_version == "1.0" def test_msgpack_rejects_in_memory_lazyexpr(): @@ -284,7 +283,7 @@ def test_msgpack_rejects_plain_python_lazyudf(tmp_path): udf = _make_persistent_python_lazyudf(tmp_path) oarr = blosc2.ObjectArray() - with pytest.raises(TypeError, match="DSLKernel"): + with pytest.raises(TypeError, match="PortableKernel"): oarr.append(udf) diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py new file mode 100644 index 000000000..2df5b85f3 --- /dev/null +++ b/tests/test_portable_artifact.py @@ -0,0 +1,768 @@ +####################################################################### +# Copyright (c) 2026, Blosc Development Team +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### +"""Portable host integration, with separate semantic and execution matrices. + +The exhaustive shared DSL corpus is certified by native miniexpr CTest. Here, +boundary values live in compact dtype/operator scenarios; engine/length checks +are independent. Rejected features do not multiply across unused JIT/size axes. +""" + +import ast +import json +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +import blosc2 + +SCALE = 2.0 +BIAS = -1.0 + + +def assert_execution_mode(kernel, jit): + # Draft 1.0 is interpreter-first, including optional JIT requests. + assert kernel.schema_version == "1.0" + assert not kernel.has_jit + + +@pytest.mark.parametrize( + ("expression", "input_dtype", "output_dtype", "expected"), + [ + ("sin(x)", "float64", "float64", np.sin([-2.0, 0.0, 2.0])), + ("cos(x)", "float32", "float32", np.cos(np.array([-2, 0, 2], dtype="float32"))), + ("int(x + 0.25)", "float64", "int64", [-1, 0, 2]), + ("float(float(x))", "float32", "float32", [-2, 0, 2]), + ("bool((x + 1.0) - x)", "float32", "bool", [True, True, True]), + ("(x + 1.0) > x", "float32", "bool", [True, True, True]), + ("(x / 2) > 0", "float64", "bool", [False, False, True]), + ("x + 0.5", "int64", "int64", [-1, 0, 2]), + ("x == 9007199254740993.0", "int64", "bool", [False, False, False]), + ("float(x)", "int64", "float32", [-2, 0, 2]), + ("float(-(x * 0.1))", "float32", "float32", [0.2, -0.0, -0.2]), + ("bool(int(x + 0.25))", "float64", "bool", [True, False, True]), + ("float(int(x) / 2)", "float32", "float32", [-1, 0, 1]), + ("int(x) / 2", "float64", "int64", [-1, 0, 1]), + ], +) +def test_typed_arithmetic_without_corpus(native_artifacts, expression, input_dtype, output_dtype, expected): + source = f"def k(x):\n return {expression}\n" + assert blosc2.validate_portable_dsl(source, {"x": input_dtype}, output_dtype)["valid"] + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=True) + assert_execution_mode(kernel, True) + actual = kernel.evaluate({"x": np.array([-2, 0, 2], dtype=input_dtype)}) + np.testing.assert_allclose(actual, np.array(expected, dtype=output_dtype), rtol=1e-6) + + +@blosc2.dsl_kernel +def affine(x): + return x * SCALE + BIAS + + +@pytest.fixture +def native_artifacts(): + from blosc2 import blosc2_ext + + available = getattr(blosc2_ext, "portable_artifact_available", lambda: False) + if not available(): + pytest.skip("Rebuild with miniexpr artifact support enabled") + + +def test_export_capture_snapshot(native_artifacts, monkeypatch): + artifact = affine.export({"x": "float64"}, "float64", metadata={"producer": "test"}) + assert artifact == affine.export({"x": "float64"}, "float64", metadata={"producer": "test"}) + decoded = json.loads(artifact) + assert "SCALE" not in decoded["source"] + assert "BIAS" not in decoded["source"] + assert len(decoded["constants"]) == 2 + monkeypatch.setitem(affine.func.__globals__, "SCALE", 99.0) + monkeypatch.setitem(affine.func.__globals__, "BIAS", 10.0) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + x = np.arange(600, dtype=np.float64).reshape(20, 30) + np.testing.assert_array_equal(kernel.evaluate({"x": x}), 2 * x - 1) + assert not kernel.has_jit + assert kernel.entry_point == "affine" + assert kernel.output_dtype == np.dtype("float64") + assert kernel.input_dtypes == {"x": np.dtype("float64")} + assert kernel.to_json() == artifact + with pytest.raises(TypeError): + kernel.input_dtypes["x"] = np.dtype("int64") + + +def test_capture_closure_and_frontend_sugar(native_artifacts): + scale = np.float32(2.0) + + @blosc2.dsl_kernel + def trig(x): + return np.sin(x) * scale + + kernel = blosc2.PortableKernel.from_json(trig.export({"x": "float32"}, "float32"), jit=False) + values = np.array([0, 1, 2], dtype="float32") + np.testing.assert_allclose(kernel.evaluate({"x": values}), np.sin(values) * scale, rtol=1e-6) + + +def test_capture_collision_and_comments(native_artifacts): + scale = 2.0 + + @blosc2.dsl_kernel + def collision(x): + """scale must not be rewritten in strings.""" + _capture_0 = x # scale stays in this comment + return _capture_0 * scale + + artifact = collision.export({"x": "float64"}, "float64") + decoded = json.loads(artifact) + assert decoded["constants"][0]["name"] == "_capture_1" + assert "# scale stays in this comment" in decoded["source"] + assert '"""scale must not be rewritten in strings."""' in decoded["source"] + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + np.testing.assert_array_equal(kernel.evaluate({"x": np.arange(3, dtype=np.float64)}), [0, 2, 4]) + + +@pytest.mark.parametrize( + ("dtype", "value", "encoding", "encoded"), + [ + ("bool", True, "boolean", True), + ("int32", np.int32(-(2**31)), "decimal", "-2147483648"), + ("int64", 2**63 - 1, "decimal", "9223372036854775807"), + ("float32", np.float32(-0.0), "ieee754-hex", "80000000"), + ("float64", -0.0, "ieee754-hex", "8000000000000000"), + ("float64", np.inf, "ieee754-hex", "7ff0000000000000"), + ("float64", np.nan, "ieee754-hex", "7ff8000000000000"), + ], +) +def test_constant_encodings(native_artifacts, dtype, value, encoding, encoded): + source = "# me:compiler=cc\ndef constant(value):\n return value\n" + author = blosc2.DSLKernel.from_source(source) + artifact = author.export({}, dtype, constants={"value": value}) + scalar = json.loads(artifact)["constants"][0] + assert scalar == {"name": "value", "dtype": dtype, "encoding": encoding, "value": encoded} + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + assert kernel.source == source + result = kernel.evaluate({}, shape=(600,)) + assert result.dtype == np.dtype(dtype) + np.testing.assert_array_equal(result, np.full(600, value, dtype=dtype)) + if encoded in ("80000000", "8000000000000000"): + assert np.signbit(result).all() + assert kernel.evaluate({}, shape=(0,)).shape == (0,) + with pytest.raises(ValueError, match="explicit shape"): + kernel.evaluate({}) + + +def test_explicit_capture_dtype(native_artifacts): + integer = 2 + + @blosc2.dsl_kernel + def k(x): + return x * integer + + inferred = json.loads(k.export({"x": "float64"}, "float64")) + assert inferred["constants"][0]["dtype"] == "int64" + artifact = k.export({"x": "float64"}, "float64", capture_dtypes={"integer": "float64"}) + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(artifact, jit=False).evaluate({"x": np.array([3.0])}), [6.0] + ) + with pytest.raises(blosc2.PortableArtifactError, match="Unused capture dtype"): + k.export({"x": "float64"}, "float64", capture_dtypes={"unused": "float64"}) + + +@pytest.mark.parametrize("value", [np.array(2.0), np.arange(2), object(), "hello"]) +def test_capture_value_types(native_artifacts, value): + capture = value + + if type(value) is str: + + @blosc2.dsl_kernel + def string_snapshot(x): + return capture + + artifact = string_snapshot.export({"x": "float64"}, "U5") + constant = json.loads(artifact)["constants"][0] + assert constant["dtype"] == "unicode32" + assert constant["encoding"] == "unicode32be-hex" + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(artifact).evaluate({"x": np.zeros(2)}), ["hello", "hello"] + ) + return + + @blosc2.dsl_kernel + def k(x): + return x + capture + + with pytest.raises(blosc2.PortableArtifactError, match="Captures"): + k.export({"x": "float64"}, "float64") + + +def test_never_calls_capture_hooks_or_functions(native_artifacts): + class Trap: + def __float__(self): + pytest.fail("capture __float__ executed") + + def item(self): + pytest.fail("capture item executed") + + capture = Trap() + + @blosc2.dsl_kernel + def k(x): + return x + capture + + with pytest.raises(blosc2.PortableArtifactError, match="Captures"): + k.export({"x": "float64"}, "float64", capture_dtypes={"capture": "float64"}) + + def callback(x): + pytest.fail("callback executed") + + @blosc2.dsl_kernel + def external(x): + return callback(x) + + with pytest.raises(blosc2.PortableArtifactError, match="callback"): + external.export({"x": "float64"}, "float64") + + +def test_shadowed_numpy_alias_is_not_rewritten(native_artifacts): + @blosc2.dsl_kernel + def local_alias(x): + np = x + return np.sin(x) + + assert "np.sin" in local_alias.dsl_source + with pytest.raises(blosc2.PortableArtifactError): + local_alias.export({"x": "float64"}, "float64") + + +@pytest.mark.parametrize("jit", [False, None, True]) +def test_artifact_backend_and_repeated_calls(native_artifacts, jit): + from concurrent.futures import ThreadPoolExecutor + + artifact = affine.export({"x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + arrays = [np.arange(600, dtype=np.float64) + i for i in range(4)] + if blosc2.IS_WASM: + # Pyodide does not provide Python threads. Keep backend selection and + # repeated-handle coverage without claiming concurrency on that host. + results = [kernel.evaluate({"x": array}) for array in arrays] + else: + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(lambda array: kernel.evaluate({"x": array}), arrays)) + for array, result in zip(arrays, results, strict=True): + np.testing.assert_array_equal(result, 2 * array - 1) + + +def test_capture_range_checks(native_artifacts): + kernel = blosc2.DSLKernel.from_source("def k(c):\n return c\n") + for value, dtype in [(2**63, "int64"), (2**31, "int32"), (1e100, "float32")]: + with pytest.raises(blosc2.PortableArtifactError): + kernel.export({}, dtype, constants={"c": value}, capture_dtypes={"c": dtype}) + # Explicit float conversion rounds to nearest, rather than rejecting precision loss. + rounded = kernel.export({}, "float64", constants={"c": 2**53 + 1}, capture_dtypes={"c": "float64"}) + assert blosc2.PortableKernel.from_json(rounded).evaluate({}, shape=())[()] == float(2**53) + + +def test_nan_payload_and_explicit_float_narrowing(native_artifacts): + author = blosc2.DSLKernel.from_source("def k(c):\n return c\n") + value = np.array([0x7FF8000000000042], dtype=np.uint64).view(np.float64)[0] + artifact = author.export({}, "float64", constants={"c": value}) + assert json.loads(artifact)["constants"][0]["value"] == "7ff8000000000042" + value = 1.1 + artifact = author.export({}, "float32", constants={"c": value}, capture_dtypes={"c": "float32"}) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + assert kernel.evaluate({}, shape=())[()] == np.float32(value) + + +def test_export_bindings_and_metadata_rejections(native_artifacts): + author = blosc2.DSLKernel.from_source("def k(x):\n return x\n") + with pytest.raises(blosc2.PortableArtifactError, match="exactly one"): + author.export({}, "float64") + with pytest.raises(blosc2.PortableArtifactError, match="exactly one"): + author.export({"x": "float64"}, "float64", constants={"x": 1.0}) + with pytest.raises(blosc2.PortableArtifactError) as error: + author.export({"x": "float64"}, "float64", metadata={"bad": "\x00"}) + assert error.value.status == "invalid_artifact" + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json(b"{\xff}") + assert error.value.status == "invalid_artifact" + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json("{\ud800}") + assert error.value.status == "invalid_artifact" + + +def test_unresolved_capture_has_location(native_artifacts): + @blosc2.dsl_kernel + def k(x): + return x + MISSING_PORTABLE_CAPTURE # noqa: F821 + + with pytest.raises(blosc2.PortableArtifactError) as error: + k.export({"x": "float64"}, "float64") + assert "MISSING_PORTABLE_CAPTURE" in str(error.value) + assert error.value.status == "invalid_source" + assert error.value.line > 0 + assert error.value.column > 0 + + +def test_binding_shapes_and_storage_adaptation(native_artifacts): + source = "def subtract(x, y):\n return x - y\n" + artifact = blosc2.DSLKernel.from_source(source).export({"y": "float64", "x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + x = np.arange(12, dtype=np.float64).reshape(3, 4)[:, ::2] + y = np.zeros((3, 2), dtype=">f8") + np.testing.assert_array_equal(kernel.evaluate({"y": y, "x": x}), x) + np.testing.assert_array_equal(kernel.evaluate({"y": y, "x": blosc2.asarray(x)}), x) + with pytest.raises(blosc2.PortableArtifactError, match="dtype"): + kernel.evaluate({"x": x.astype(np.float32), "y": y}) + with pytest.raises(blosc2.PortableArtifactError, match="shapes"): + kernel.evaluate({"x": x, "y": y.ravel()}) + with pytest.raises(blosc2.PortableArtifactError, match="Missing or extra"): + kernel.evaluate({"x": x}) + empty = np.empty((0, 3), dtype=np.float64) + assert kernel.evaluate({"y": empty, "x": empty}).shape == (0, 3) + + +def test_import_is_native_only(native_artifacts, monkeypatch): + artifact = affine.export({"x": "float64"}, "float64") + + def forbidden(*args, **kwargs): + pytest.fail("Python source/JSON preparation was invoked") + + with monkeypatch.context() as context: + context.setattr(ast, "parse", forbidden) + context.setattr(json, "loads", forbidden) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + result = kernel.evaluate({"x": np.array([0.0, 1.0, 2.0, 3.0])}) + np.testing.assert_array_equal(result, [-1, 1, 3, 5]) + + +def test_export_runs_in_standalone_c(native_artifacts, tmp_path): + runner_path = os.environ.get("MINIEXPR_ARTIFACT_RUNNER") + if not runner_path: + pytest.skip("Set MINIEXPR_ARTIFACT_RUNNER to the standalone native runner") + runner = Path(runner_path) + if not runner.is_file(): + pytest.skip("Set MINIEXPR_ARTIFACT_RUNNER to the standalone native runner") + artifact = tmp_path / "export.json" + artifact.write_text(affine.export({"x": "float64"}, "float64")) + result = subprocess.run([str(runner), str(artifact), "off"], check=True, capture_output=True, text=True) + assert result.stdout.splitlines() == ["jit=0", "-1", "1", "3", "5"] + # A fresh interpreter imports only blosc2, never this authoring test module. + program = "import pathlib, blosc2, numpy as np; " + program += ( + f"k = blosc2.PortableKernel.from_json(pathlib.Path({str(artifact)!r}).read_bytes(), jit=False); " + ) + program += "print(k.evaluate({'x': np.array([0., 1., 2., 3.])}).tolist())" + result = subprocess.run([sys.executable, "-c", program], check=True, capture_output=True, text=True) + assert result.stdout.strip() == "[-1.0, 1.0, 3.0, 5.0]" + + +def test_native_diagnostics_and_runtime_errors(native_artifacts): + artifact = affine.export({"x": "float64"}, "float64") + changed = artifact.replace('"schema_version":"1.0"', '"schema_version":"99"') + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json(changed, jit=False) + assert error.value.status == "unsupported_requirement" + for field in ('"schema_version":"1.0"', '"version":"1.0"'): + with pytest.raises(blosc2.PortableArtifactError, match=r"unsupported.*version"): + blosc2.PortableKernel.from_json(artifact.replace(field, field.replace("1.0", "0.1"))) + retired_manifest = json.loads(artifact) + retired_manifest["schema_version"] = retired_manifest["language"]["version"] = "0.1" + retired_manifest["requires"] = ["core-scalar"] + retired_manifest["output"]["contract"] = "scalar-per-element" + retired_manifest.pop("context") + # Reject the actual retired envelope before applying 1.0's required fields. + with pytest.raises(blosc2.PortableArtifactError, match=r"unsupported.*version"): + blosc2.PortableKernel.from_json(json.dumps(retired_manifest)) + with pytest.raises(blosc2.PortableArtifactError) as retired: + affine.export({"x": "float64"}, "float64", version="0.1") + assert retired.value.status == "unsupported_version" + changed = artifact.replace('"schema_version":', '"schema_version":"0.1","schema_version":') + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json(changed, jit=False) + assert error.value.status == "invalid_artifact" + source = "def partial(x):\n if x > 0:\n return x\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + with pytest.raises(blosc2.PortableArtifactError) as error: + kernel.evaluate({"x": np.array([-1.0])}) + assert error.value.status == "evaluation_error" + assert error.value.native_status != 0 + assert kernel.evaluate({"x": np.empty(0, dtype=np.float64)}).size == 0 + + +def test_import_missing_native_support(monkeypatch): + from blosc2 import blosc2_ext + + monkeypatch.delattr(blosc2_ext, "PortableArtifactHandle", raising=False) + with pytest.raises(NotImplementedError, match="artifact support"): + blosc2.PortableKernel.from_json("{}") + + +def test_optional_adapter_availability(): + from blosc2 import blosc2_ext + + available = getattr(blosc2_ext, "portable_artifact_available", lambda: False)() + if available: + with pytest.raises(blosc2.PortableArtifactError, match="invalid_artifact"): + blosc2.PortableKernel.from_json("{}", jit=False) + else: + with pytest.raises(NotImplementedError, match="artifact support"): + blosc2.PortableKernel.from_json("{}", jit=False) + + +# Execution engines and sizes are independent of the numeric scenario matrix. +# Exercise scalar, short-vector, and tile-tail execution, including repeated +# handles, Boolean bytes, float32 rounding, and local negative zero. +@pytest.mark.parametrize( + ("compiler", "jit"), + [ + pytest.param("tcc", False, id="interpreter"), + pytest.param("tcc", True, id="tcc"), + pytest.param("cc", True, id="cc"), + ], +) +@pytest.mark.parametrize("count", [1, 2, 257]) +def test_numeric_execution_shapes(native_artifacts, compiler, jit, count): + cases = [ + ("bool", "bool", " half = x * 0.5\n return half\n", [True, False], [True, False]), + ( + "float32", + "float32", + " rounded = float(x) + 0.1\n return rounded - float(x)\n", + [2**24, -(2**24)], + [ + np.float32((float(2**24) + 0.1) - float(2**24)), + np.float32((float(-(2**24)) + 0.1) - float(-(2**24))), + ], + ), + ("float64", "float64", " truth = bool(x)\n return -truth\n", [0.0, 1.0], [0.0, -1.0]), + ] + for input_dtype, output_dtype, body, samples, expected in cases: + source = f"# me:compiler={compiler}\ndef k(x):\n{body}" + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + for _ in range(2): + actual = kernel.evaluate({"x": np.resize(np.array(samples, dtype=input_dtype), count)}) + desired = np.resize(np.array(expected, dtype=output_dtype), count) + np.testing.assert_array_equal(actual, desired) + if output_dtype == "bool": + assert np.all(actual.view("uint8") <= 1) + else: + np.testing.assert_array_equal(np.signbit(actual), np.signbit(desired)) + + +@pytest.mark.parametrize( + ("body", "samples"), + [ + (" return x + 0.5\n", [True, True]), + (" return x * 0.5\n", [False, True]), + (" return bool(x + 0.5)\n", [True, True]), + (" return (x + x) == 2\n", [False, True]), + (" return (x * 0.5) == 0.5\n", [False, True]), + (" return int(x + 0.5) == 0\n", [True, False]), + (" half = x * 0.5\n return half == 0.5\n", [False, True]), + (" half = x * 0.5\n return half * 2 == x\n", [True, True]), + (" return x - 1\n", [True, False]), + (" return -x\n", [False, True]), + (" return (x + 9007199254740992) - 9007199254740992\n", [False, True]), + ], +) +def test_bool_output_numeric_arithmetic(native_artifacts, body, samples): + compiler, jit, count = "tcc", False, 257 + artifact = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n{body}").export( + {"x": "bool"}, "bool" + ) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + values = np.resize(np.array([False, True]), count) + expected = np.resize(np.array(samples), count) + for _ in range(2): + actual = kernel.evaluate({"x": values}) + np.testing.assert_array_equal(actual, expected) + assert actual.dtype == np.dtype("bool") + assert np.all(actual.view("uint8") <= 1) + + +# Test each dtype/operation once, rotating the independent execution paths and +# output widths. Backend/length coverage is tested separately below, not crossed +# with every numeric scenario. Locals and direct expressions both remain covered. +@pytest.mark.parametrize( + ("input_dtype", "operation", "compiler", "jit", "output_dtype", "local"), + [ + pytest.param( + dtype, + operation, + "cc" if (d + o) % 3 == 2 else "tcc", + (d + o) % 3 != 0, + "float32" if (d + o) % 2 == 0 else "float64", + (d // 2 + o) % 2 == 1, + id=f"{dtype}-{operation}", + ) + for d, dtype in enumerate(["bool", "int32", "int64", "float32", "float64"]) + for o, operation in enumerate(["sum", "difference", "product", "negation"]) + ], +) +def test_bool_cast_floating_arithmetic( + native_artifacts, compiler, jit, input_dtype, output_dtype, operation, local +): + count = 257 + expressions = { + "sum": "bool(x) + bool(x)", + "difference": "bool(x) - bool(x)", + "product": "bool(x) * bool(x)", + "negation": "-bool(x)", + } + expression = expressions[operation] + body = "" + if local: + expression = expression.replace("bool(x)", "truth") + body = " truth = bool(x)\n" + source = f"# me:compiler={compiler}\ndef k(x):\n{body} return {expression}\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + samples = [0, 1, -1, 2] + if input_dtype == "int64": + samples += [2**53 + 1, -(2**53 + 1)] + elif input_dtype.startswith("float"): + samples += [-0.0, np.finfo(input_dtype).smallest_subnormal, np.inf, -np.inf, np.nan] + values = np.resize(np.array(samples, dtype=input_dtype), count) + truth = (values != 0).astype("int64") + expected = { + "sum": truth + truth, + "difference": truth - truth, + "product": truth * truth, + "negation": -truth, + }[operation].astype(output_dtype) + for _ in range(2): + actual = kernel.evaluate({"x": values}) + np.testing.assert_array_equal(actual, expected) + zeros = expected == 0 + np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) + + +@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) +def test_float_cast_arithmetic_rounding(native_artifacts, output_dtype): + compiler, jit, count = "tcc", False, 257 + source = f"# me:compiler={compiler}\ndef k(x):\n return (float(x) + 0.1) - float(x)\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + samples = np.array( + [ + 2**24, + -(2**24), + 0.0, + -0.0, + np.finfo("float32").smallest_subnormal, + np.finfo("float32").max, + np.inf, + -np.inf, + np.nan, + 0.1, + ], + dtype="float32", + ) + values = np.resize(samples, count) + computation = values.astype("float64") + constant = np.float64(0.1) + with np.errstate(all="ignore"): + expected = ((computation + constant) - computation).astype(output_dtype) + for _ in range(2): + actual = kernel.evaluate({"x": values}) + np.testing.assert_array_equal(actual, expected) + zeros = expected == 0 + np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) + + +@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) +@pytest.mark.parametrize("operation", ["add", "subtract", "multiply"]) +def test_float_arithmetic_constant_rounding(native_artifacts, output_dtype, operation): + compiler, jit, count = "cc", True, 257 + expressions = { + "add": "(x + 0.1) - x", + "subtract": "x - 0.1", + "multiply": "(x * 0.1) - x", + } + source = f"# me:compiler={compiler}\n# me:fp=strict\ndef k(x):\n return {expressions[operation]}\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + values = np.resize(np.array([0.1, -1, 0, 1, 16777216], dtype="float32"), count) + # Operand-driven float32 computation precedes the output conversion. + arithmetic_values = values + constant = np.array(0.1, dtype="float32") + if operation == "add": + expected = (arithmetic_values + constant) - arithmetic_values + elif operation == "subtract": + expected = arithmetic_values - constant + else: + expected = (arithmetic_values * constant) - arithmetic_values + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected.astype(output_dtype)) + + +@pytest.mark.parametrize(("compiler", "jit"), [("tcc", False), ("tcc", True), ("cc", True)]) +@pytest.mark.parametrize("cap", ["3", "0", "-1", "invalid"]) +def test_while_cap_policy(native_artifacts, monkeypatch, compiler, jit, cap): + count = 257 + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", cap) + source = f"# me:compiler={compiler}\ndef k(x):\n n = 0\n while n < x:\n n = n + 1\n return n\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + for target in [0, 2, 3, 4]: + values = np.full(count, target, dtype="int64") + if cap == "3" and target == 4: + with pytest.raises(blosc2.PortableArtifactError) as error: + kernel.evaluate({"x": values}) + assert error.value.status == "evaluation_error" + else: + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values) + # Failed output contents are unspecified; the same prepared handle remains usable. + values = np.full(count, 2, dtype="int64") + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values) + assert kernel.evaluate({"x": np.empty(0, dtype="int64")}).size == 0 + + +@pytest.mark.parametrize(("compiler", "jit"), [("tcc", False), ("tcc", True), ("cc", True)]) +@pytest.mark.parametrize("compile_cap", ["0", "3"]) +def test_while_cap_change_after_loading(native_artifacts, monkeypatch, compiler, jit, compile_cap): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", compile_cap) + source = f"# me:compiler={compiler}\ndef k(x):\n n = 0\n while n < x:\n n = n + 1\n return n\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + values = np.array([4], dtype="int64") + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "2") + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + kernel.evaluate({"x": values}) + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "4") + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values) + # The identical source loaded under a different cap must not reuse code + # compiled with the old cap from the shared JIT cache. + new_kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(new_kernel, jit) + np.testing.assert_array_equal(new_kernel.evaluate({"x": values}), values) + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + kernel.evaluate({"x": values}) + + +@pytest.mark.parametrize( + ("dtype", "compiler", "jit"), + [("int32", "tcc", False), ("int64", "tcc", True), ("float32", "cc", True), ("float64", "tcc", True)], +) +def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dtype): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + source = ( + f"# me:compiler={compiler}\ndef while_cap_chain(x):\n" + " n = 0\n while 0 <= n < x:\n n = n + 1\n return n\n" + ) + artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + # Lane ordering is the actual regression boundary, not every size/backend + # combination. Keep both first-lane states in each dtype scenario. + for samples in [[0, 2, 3, -1], [3, 2, 0, -1]]: + values = np.resize(np.array(samples, dtype=dtype), 257) + expected = np.maximum(values, np.array(0, dtype=dtype)) + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + + +@pytest.mark.parametrize( + ("dtype", "compiler", "jit"), + [("int32", "tcc", False), ("int64", "tcc", True), ("float32", "cc", True), ("float64", "tcc", True)], +) +@pytest.mark.parametrize("case", ["masked_local_chain", "masked_bool_chain"]) +def test_masked_local_constant_chain(native_artifacts, compiler, jit, dtype, case): + count = 257 + bodies = { + "masked_local_chain": " n = 0\n if x == 0:\n n = 1\n return 0 <= n < x\n", + "masked_bool_chain": ( + " flag = bool(0)\n if x == 0:\n flag = bool(1)\n" + " return bool(0) <= flag < bool(x)\n" + ), + } + source = f"# me:compiler={compiler}\ndef {case}(x):\n{bodies[case]}" + artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, "bool") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + values = np.resize(np.array([0, 1, 2, -1], dtype=dtype), count) + expected = np.resize(np.array([False, True, True, case == "masked_bool_chain"]), count) + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + + +# Cap policy and post-load changes above cover all engines. These scenarios +# isolate control-flow semantics rather than crossing the same policy again. +@pytest.mark.parametrize( + ("body", "expected", "compiler", "jit"), + [ + (" n = 0\n while 0 <= n < 3:\n n = n + 1\n return n\n", 3, "tcc", False), + ( + " n = 0\n while x:\n n = n + 1\n if n == 3:\n break\n return n\n", + 3, + "tcc", + True, + ), + ( + " n = 0\n while x:\n n = n + 1\n if n == 3:\n return n\n return n\n", + 3, + "cc", + True, + ), + ( + " total = 0\n for i in range(2):\n n = 0\n while n < 3:\n" + " n = n + 1\n total = total + 1\n return total\n", + 6, + "tcc", + True, + ), + ( + " total = 0\n n = 0\n while n < 3:\n n = n + 1\n m = 0\n" + " while m < 3:\n m = m + 1\n total = total + 1\n return total\n", + 9, + "cc", + True, + ), + (" if x > 1:\n while x:\n pass\n return x\n", 1, "tcc", False), + ], +) +def test_while_cap_control_flow(native_artifacts, monkeypatch, compiler, jit, body, expected): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + artifact = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n{body}").export( + {"x": "int64"}, "int64" + ) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert_execution_mode(kernel, jit) + np.testing.assert_array_equal( + kernel.evaluate({"x": np.ones(5, dtype="int64")}), np.full(5, expected, dtype="int64") + ) + + +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("expression", ["int(x)", "bool(x)"]) +def test_value_cast_large_integer(native_artifacts, jit, expression): + source = f"# me:compiler=tcc\ndef k(x):\n return {expression}\n" + output_dtype = "int64" if expression == "int(x)" else "bool" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + values = np.array([-(2**53 + 1), 0, 2**53 + 1, 2**63 - 1], dtype="int64") + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values.astype(output_dtype)) + + +@pytest.mark.parametrize("jit", [False, True]) +def test_value_cast_nonfinite_truth(native_artifacts, jit): + source = "# me:compiler=tcc\ndef k(x):\n return bool(x)\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, "bool") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + values = np.array([-0.0, 0.0, np.nan, np.inf, -np.inf, 5e-324]) + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values.astype("bool")) diff --git a/tests/test_portable_descriptor.py b/tests/test_portable_descriptor.py new file mode 100644 index 000000000..58a5d7caf --- /dev/null +++ b/tests/test_portable_descriptor.py @@ -0,0 +1,141 @@ +"""Opt-in installed-native descriptor fixtures; never search a sibling checkout.""" + +import json + +import numpy as np +import pytest + +from blosc2 import blosc2_ext + +pytestmark = pytest.mark.skipif( + not getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(), + reason="Installed native runtime has no draft 1.0 descriptor ABI", +) + + +def manifest(source, inputs, output, *, requires, ndim=0): + return json.dumps( + { + "schema_version": "1.0", + "language": {"name": "miniexpr", "version": "1.0"}, + "requires": requires, + "source": source, + "entry_point": "k", + "inputs": inputs, + "constants": [], + "output": output, + "semantics": {"fp": "strict"}, + "context": {"ndim": ndim}, + "metadata": {"status": "implementation-draft"}, + } + ).encode() + + +def test_descriptor_scalar_mask_empty(): + artifact = blosc2_ext.PortableArtifactHandle( + manifest( + "def k(x):\n return sum(x)\n", + [{"name": "x", "dtype": "int64"}], + {"dtype": "int64", "contract": "block_scalar"}, + requires=["numeric", "block-reductions"], + ), + 0, + ) + assert artifact.descriptor_info()["cardinality"] == "block_scalar" + result = artifact.evaluate_block( + {"x": np.array([np.iinfo(np.int64).max, 2, 3])}, + (3,), + valid_mask=np.array([False, True, True]), + ) + assert result.shape == () + assert result == 5 + assert artifact.evaluate_block({"x": np.array([], dtype="int64")}, (0,)) == 0 + assert ( + artifact.evaluate_block( + {"x": np.array([np.iinfo(np.int64).max, 2, 3])}, + (3,), + valid_mask=np.zeros(3, dtype=bool), + ) + == 0 + ) + record = json.loads( + manifest( + "def k(x):\n s = sum(x)\n return s\n", + [{"name": "x", "dtype": "int64"}], + {"dtype": "int64", "contract": "block_scalar"}, + requires=["numeric", "block-reductions"], + ) + ) + local = blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + assert ( + local.evaluate_block( + {"x": np.array([99, 2, 3], dtype="int64")}, (3,), valid_mask=np.array([False, True, True]) + ) + == 5 + ) + record["output"]["contract"] = "elementwise" + with pytest.raises(ValueError, match="cardinality"): + blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + record["output"]["contract"] = "block_scalar" + record["requires"].append("control-flow") + for body in ( + " if x < 0:\n return s + 1\n return s\n", + " for i in range(x):\n return s\n return s + 1\n", + " for i in range(3):\n if x < 0:\n break\n return s + 1\n return s\n", + " for i in range(3):\n if x < 0:\n continue\n return s + 1\n return s\n", + " while all(x != 0):\n if x < 0:\n break\n return s + 1\n return s\n", + " if s < 0:\n return s\n elif x < 0:\n return s + 1\n return s\n", + ): + record["source"] = "def k(x):\n s = sum(x)\n" + body + with pytest.raises(ValueError, match="ambiguous block-scalar"): + blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + for body in ( + " if all(x > 0):\n return s + 1\n return s\n", + " for i in range(3):\n if s > 0:\n return s + i\n return s\n", + " for i in range(3):\n if x < 0:\n break\n return s\n", + " while all(x > 0):\n return s + 1\n return s\n", + ): + record["source"] = "def k(x):\n s = sum(x)\n" + body + coherent = blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + assert coherent.evaluate_block({"x": np.array([-1, 2], dtype="int64")}, (2,)) == 1 + + +def test_descriptor_fixed_unicode(): + artifact = blosc2_ext.PortableArtifactHandle( + manifest( + "def k(x):\n return upper(x)\n", + [{"name": "x", "dtype": "unicode32", "itemsize": 16}], + {"dtype": "unicode32", "itemsize": 16, "contract": "elementwise"}, + requires=["numeric", "fixed-strings"], + ), + 0, + ) + result = artifact.evaluate_block({"x": np.array(["ß a"], dtype="U4")}, (1,)) + np.testing.assert_array_equal(result, np.array(["SS A"], dtype="U4")) + record = json.loads( + manifest( + "def k(x):\n return upper(x)\n", + [{"name": "x", "dtype": "unicode32", "itemsize": 16}], + {"dtype": "unicode32", "itemsize": 12, "contract": "elementwise"}, + requires=["numeric", "fixed-strings"], + ) + ) + with pytest.raises(ValueError, match="width"): + blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + + +def test_descriptor_logical_coordinates(): + artifact = blosc2_ext.PortableArtifactHandle( + manifest( + "def k():\n return _flat_idx\n", + [], + {"dtype": "int64", "contract": "elementwise"}, + requires=["numeric", "nd-context"], + ndim=2, + ), + 0, + ) + result = artifact.evaluate_block({}, (2, 2), logical_shape=(4, 5), block_origin=(2, 3)) + np.testing.assert_array_equal(result, [[13, 14], [18, 19]]) + with pytest.raises(ValueError, match="Explicit logical"): + artifact.evaluate_block({}, (2, 2)) diff --git a/tests/test_portable_descriptor_host.py b/tests/test_portable_descriptor_host.py new file mode 100644 index 000000000..a580ca18b --- /dev/null +++ b/tests/test_portable_descriptor_host.py @@ -0,0 +1,58 @@ +"""Host routing tests run against the installed pin without a native 1.0 engine.""" + +import numpy as np +import pytest + +from blosc2 import PortableKernel, blosc2_ext + + +def test_native_descriptor_authority_and_block_routing(monkeypatch): + calls = [] + marker = object() + + class Handle: + def __init__(self, record, jit): + assert record == b"opaque record validated only by native code" + + def info(self): + return { + "source": "native source", + "entry_point": "k", + "inputs": {}, + "output_dtype": None, + "jit": False, + } + + def descriptor_info(self): + return { + "schema_version": "1.0", + "inputs": {"x": np.dtype("int64")}, + "output_dtype": np.dtype("int64"), + "cardinality": "block_scalar", + "ndim": 2, + } + + def evaluate_block(self, arrays, shape, **context): + calls.append((arrays, shape, context)) + return marker + + monkeypatch.setattr(blosc2_ext, "PortableArtifactHandle", Handle) + monkeypatch.setattr(blosc2_ext, "portable_descriptor_available", lambda: True, raising=False) + kernel = PortableKernel.from_json(b"opaque record validated only by native code") + assert kernel.result_cardinality == "block_scalar" + assert kernel.context_ndim == 2 + with pytest.raises(ValueError, match="explicit logical_shape"): + kernel.evaluate({"x": np.arange(4, dtype="int64").reshape(2, 2)}) + values = np.arange(8, dtype=">i8").reshape(2, 4)[:, ::2] + mask = np.array([[True, False], [True, True]]) + assert ( + kernel.evaluate_block({"x": values}, logical_shape=(4, 5), block_origin=(2, 3), valid_mask=mask) + is marker + ) + arrays, shape, context = calls[0] + assert shape == (2, 2) + assert arrays["x"].flags.c_contiguous + assert arrays["x"].dtype.isnative + assert context["logical_shape"] == (4, 5) + assert context["block_origin"] == (2, 3) + assert context["valid_mask"] is mask diff --git a/tests/test_portable_lazy.py b/tests/test_portable_lazy.py new file mode 100644 index 000000000..6ccee5a50 --- /dev/null +++ b/tests/test_portable_lazy.py @@ -0,0 +1,148 @@ +import numpy as np +import pytest +from test_portable_descriptor import manifest + +import blosc2 +from blosc2 import blosc2_ext + +pytestmark = pytest.mark.skipif( + not getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(), + reason="Installed runtime has no draft descriptor ABI", +) + + +def kernel(source, *, scalar=False, nd=False, string=False): + dtype = "unicode32" if string else "int64" + inputs = [{"name": "x", "dtype": dtype}] + output = {"dtype": dtype, "contract": "block_scalar" if scalar else "elementwise"} + requires = ["numeric"] + if string: + inputs[0]["itemsize"] = output["itemsize"] = 16 + requires.append("fixed-strings") + if scalar: + requires.append("block-reductions") + if nd: + requires.append("nd-context") + return blosc2.PortableKernel.from_json( + manifest(source, inputs, output, requires=requires, ndim=2 if nd else 0), jit=False + ) + + +@pytest.mark.parametrize("scalar", [False, True]) +def test_logical_groups_partial_and_rechunk(tmp_path, scalar): + values = np.arange(35, dtype="int64").reshape(5, 7) + stored = blosc2.asarray(values, chunks=(3, 4), blocks=(1, 2), urlpath=tmp_path / "input.b2nd") + k = kernel( + "def k(x):\n return sum(x)\n" if scalar else "def k(x):\n return x + _flat_idx\n", + scalar=scalar, + nd=not scalar, + ) + lazy = k.lazy({"x": stored}, partitions=(2, 3)) + expected = ( + np.array([[values[i : i + 2, j : j + 3].sum() for j in range(0, 7, 3)] for i in range(0, 5, 2)]) + if scalar + else values * 2 + ) + assert lazy.partitions == (2, 3) + np.testing.assert_array_equal(lazy[:], expected) + np.testing.assert_array_equal(lazy[1:, ::2], expected[1:, ::2]) + np.testing.assert_array_equal( + lazy.compute(item=(slice(1, None), slice(None, None, 2)))[:], expected[1:, ::2] + ) + np.testing.assert_array_equal(lazy.rechunk(chunks=(1, 2), blocks=(1, 1))[:], expected) + path = tmp_path / "lazy.b2nd" + lazy.save(path) + reopened = blosc2.open(path) + assert reopened.partitions == (2, 3) + np.testing.assert_array_equal(reopened[:], expected) + mixed = lazy + 3 + mixed.save(tmp_path / "mixed.b2nd") + np.testing.assert_array_equal(blosc2.open(tmp_path / "mixed.b2nd")[:], expected + 3) + + +def test_string_roundtrip_and_native_only_load(tmp_path, monkeypatch): + values = np.array(["ß a", "ab", "z"], dtype="U4") + stored = blosc2.asarray(values, urlpath=tmp_path / "text.b2nd") + lazy = kernel("def k(x):\n return upper(x)\n", string=True).lazy({"x": stored}, partitions=(2,)) + lazy.save(tmp_path / "textlazy.b2nd") + import importlib + + dsl_module = importlib.import_module("blosc2.dsl_kernel") + monkeypatch.setattr( + dsl_module, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + import blosc2.b2objects as b2objects + + monkeypatch.setattr( + b2objects, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + np.testing.assert_array_equal(blosc2.open(tmp_path / "textlazy.b2nd")[:], ["SS A", "AB", "Z"]) + + +def test_preflight_preserves_destination(tmp_path): + lazy = kernel("def k(x):\n return x + 1\n").lazy({"x": np.arange(3, dtype="int64")}) + path = tmp_path / "existing" + path.write_bytes(b"sentinel") + with pytest.raises((TypeError, ValueError)): + lazy.save(path) + assert path.read_bytes() == b"sentinel" + + +@pytest.mark.parametrize("route", ["frame", "msgpack", "embed", "dict", "tree"]) +def test_container_routes(tmp_path, route): + from blosc2.msgpack_utils import msgpack_packb, msgpack_unpackb + + source = blosc2.asarray(np.arange(9, dtype="int64"), urlpath=tmp_path / "source.b2nd") + lazy = kernel("def k(x):\n return sum(x)\n", scalar=True).lazy({"x": source}, partitions=(4,)) + expected = np.array([6, 22, 8]) + if route == "frame": + reopened = blosc2.from_cframe(lazy.to_cframe()) + elif route == "msgpack": + reopened = msgpack_unpackb(msgpack_packb(lazy), deserialize="safe") + else: + cls = {"embed": blosc2.EmbedStore, "dict": blosc2.DictStore, "tree": blosc2.TreeStore}[route] + with cls(str(tmp_path / "store.b2z"), mode="w") as store: + store["/lazy"] = lazy + reopened = store["/lazy"] + np.testing.assert_array_equal(reopened[:], expected) + with cls(str(tmp_path / "store.b2z"), mode="r") as store: + reopened = store["/lazy"] + np.testing.assert_array_equal(reopened[:], expected) + return + np.testing.assert_array_equal(reopened[:], expected) + + +def test_partial_reads_only_original_groups(): + class Operand: + dtype = np.dtype("int64") + shape = (12,) + + def __init__(self): + self.reads = [] + + def __getitem__(self, item): + self.reads.append(item) + return np.arange(12, dtype="int64")[item] + + operand = Operand() + lazy = kernel("def k(x):\n return sum(x)\n", scalar=True).lazy({"x": operand}, partitions=(4,)) + np.testing.assert_array_equal(lazy[1:2], [22]) + assert operand.reads == [(slice(4, 8),)] + assert lazy[1] == 22 + np.testing.assert_array_equal(lazy[::-1], [38, 22, 6]) + author = blosc2.DSLKernel.from_source("def k(x):\n return x + sum(x)\n") + elementwise = blosc2.PortableKernel.from_json(author.export({"x": "int64"}, "int64", version="1.0")) + grouped = elementwise.lazy({"x": operand}, partitions=(4,)) + operand.reads.clear() + np.testing.assert_array_equal(grouped[7:3:-2], [29, 27]) + assert operand.reads == [(slice(4, 8),)] + np.testing.assert_array_equal(grouped[4:4], np.array([], dtype="int64")) + assert operand.reads == [(slice(4, 8),)] + + +def test_broadcast_coordinates_keep_output_domain(): + values = np.arange(7, dtype="int64").reshape(1, 7) + k = kernel("def k(x):\n return x + _flat_idx\n", nd=True) + lazy = k.lazy({"x": values}, shape=(5, 7), partitions=(2, 3)) + expected = values + np.arange(35).reshape(5, 7) + np.testing.assert_array_equal(lazy[3:, 4:], expected[3:, 4:]) diff --git a/tests/test_portable_policy.py b/tests/test_portable_policy.py new file mode 100644 index 000000000..8df12dba7 --- /dev/null +++ b/tests/test_portable_policy.py @@ -0,0 +1,155 @@ +"""Metadata policy/preflight checks independent of the installed native version.""" + +from pathlib import Path + +import numpy as np +import pytest + +import blosc2 +from blosc2.b2objects import decode_b2object_payload + + +@pytest.mark.parametrize("policy", ["safe", "full"]) +@pytest.mark.parametrize("expression", ["x + 1", "sqrt(x)"]) +def test_nested_legacy_effective_policy(tmp_path, policy, expression): + fixture = Path(__file__).parent / "data" / "legacy_lazyudf_v1" / "expr.b2nd" + carrier = blosc2.empty((5,), dtype="float64", meta={"b2o": {"kind": "lazyexpr", "version": 1}}) + carrier.schunk.vlmeta["b2o"] = { + "kind": "lazyexpr", + "version": 1, + "expression": expression, + "operands": {"x": {"kind": "urlpath", "version": 1, "urlpath": str(fixture)}}, + } + if policy == "safe": + with pytest.raises(blosc2.UnsafeDeserializationError): + blosc2.from_cframe(carrier.to_cframe(), deserialize=policy) + else: + opened = blosc2.from_cframe(carrier.to_cframe(), deserialize=policy) + nested = next(iter(opened.operands.values())) + assert isinstance(nested, blosc2.LazyUDF) + np.testing.assert_allclose(nested.compute()[:], (np.arange(5) * 3) ** 2) + + +def test_structured_legacy_rejects_before_python(monkeypatch): + from msgpack import ExtType, packb + + import blosc2.b2objects as b2objects + from blosc2.msgpack_utils import msgpack_unpackb + + record = { + "kind": "lazyudf", + "version": 1, + "function_kind": "dsl", + "dsl_version": 1, + "udf_source": "def k():\n return 1\n", + "name": "k", + "dtype": "int64", + "shape": [2], + "operands": {}, + "kwargs": {}, + } + legacy = ExtType(43, packb(record)) + nested = ExtType(46, packb({"values": [legacy], "shape": [1]})) + for payload in (packb(legacy), packb(nested)): + trusted = msgpack_unpackb(payload, deserialize="full") + assert isinstance(trusted if isinstance(trusted, blosc2.LazyUDF) else trusted[0], blosc2.LazyUDF) + legacy_recipe = trusted if isinstance(trusted, blosc2.LazyUDF) else trusted[0] + with pytest.raises(TypeError, match="legacy"): + legacy_recipe.to_cframe() + + monkeypatch.setattr(b2objects, "kernel_from_source", lambda *a, **k: pytest.fail("Policy bypass")) + with pytest.raises(blosc2.UnsafeDeserializationError): + decode_b2object_payload({"kind": "lazyudf", "version": 1}, deserialize="safe") + for payload in (packb(legacy), packb(nested)): + with pytest.raises(blosc2.UnsafeDeserializationError): + msgpack_unpackb(payload) + with pytest.raises(blosc2.UnsafeDeserializationError): + msgpack_unpackb(payload, deserialize="safe") + + +@pytest.mark.parametrize("route", ["embed", "dict", "tree"]) +def test_rejected_kernel_does_not_replace_store_value(tmp_path, route): + cls = {"embed": blosc2.EmbedStore, "dict": blosc2.DictStore, "tree": blosc2.TreeStore}[route] + udf = blosc2.lazyudf(lambda inputs, output, offset: None, (np.arange(4),), dtype="int64") + with cls(str(tmp_path / "store.b2z"), mode="w") as store: + store["/value"] = np.arange(4) + with pytest.raises(TypeError, match="PortableKernel"): + store["/value"] = udf + np.testing.assert_array_equal(store["/value"][:], np.arange(4)) + operand = blosc2.asarray(np.arange(4), urlpath=tmp_path / "input.b2nd", mode="w") + for body in ("print(x)\n return x", "return callback(x)"): + kernel = blosc2.DSLKernel.from_source(f"def invalid(x):\n {body}\n") + invalid = blosc2.lazyudf(kernel, (operand,), dtype="int64") + with pytest.raises(blosc2.PortableArtifactError): + store["/value"] = invalid + np.testing.assert_array_equal(store["/value"][:], np.arange(4)) + valid = blosc2.DSLKernel.from_source("def valid(x):\n return x + 2\n") + store["/recipe"] = blosc2.lazyudf(valid, (operand,), dtype="int64") + np.testing.assert_array_equal(store["/recipe"][:], np.arange(4) + 2) + + +def test_ctable_legacy_metadata_policy_before_reconstruction(monkeypatch): + from types import SimpleNamespace + + table = object.__new__(blosc2.CTable) + table._storage = SimpleNamespace(_deserialize_mode="safe") + schema = { + "computed_columns": [ + {"name": "bad", "col_deps": [], "dtype": "int64", "kind": "dsl", "dsl_source": "invalid"} + ] + } + with pytest.raises( + blosc2.UnsafeDeserializationError, match=r"block/batch-dependent.*deserialize='full'" + ): + table._load_computed_cols_from_schema(schema) + table._computed_cols = {"x": {"kind": "dsl"}} + table._materialized_cols = {} + with pytest.raises(TypeError, match="row-domain"): + table._preflight_portable_persistence() + + +@pytest.mark.parametrize("generated", [False, True]) +def test_legacy_table_recipes_retain_full_opt_in(tmp_path, generated): + from dataclasses import dataclass + + @dataclass + class Row: + x: float = 0.0 + + path = str(tmp_path / "legacy.b2d") + table = blosc2.CTable( + Row, urlpath=path, mode="w", new_data={"x": [2.0, 4.0]}, create_summary_index=False + ) + if generated: + table.add_column("old", blosc2.float64(), values=np.array([4.0, 6.0])) + schema = table._schema_dict_with_computed() + recipe = { + "name": "old", + "dsl_source": "def old(x):\n return x + sum(x)\n", + "col_deps": ["x"], + "dtype": "float64", + } + if generated: + recipe.update(transformer_kind="dsl", stale=False, computed_column=None, expression=None) + schema["materialized_columns"] = [recipe] + else: + recipe["kind"] = "dsl" + schema["computed_columns"] = [recipe] + table._storage.save_schema(schema) + table.close() + with pytest.raises( + blosc2.UnsafeDeserializationError, match=r"block/batch-dependent.*deserialize='full'" + ): + blosc2.CTable.open(path) + trusted = blosc2.CTable.open(path, mode="a", deserialize="full") + if generated: + expected_batch = trusted._evaluate_dsl_materialized_batch( + trusted._materialized_cols["old"], {"x": np.array([8.0])} + ) + trusted.append({"x": 8.0}) + # Appended batches retain their old groups rather than adopting the new API contract. + np.testing.assert_array_equal(trusted["old"][:], [4.0, 6.0, expected_batch[0]]) + else: + # The original two-row execution block, not independent one-lane rows. + np.testing.assert_array_equal(trusted["old"][:], [8.0, 10.0]) + trusted.close() diff --git a/tests/test_portable_table_export.py b/tests/test_portable_table_export.py new file mode 100644 index 000000000..cb15e1167 --- /dev/null +++ b/tests/test_portable_table_export.py @@ -0,0 +1,261 @@ +import importlib +from dataclasses import dataclass + +import numpy as np +import pytest + +import blosc2 +from blosc2 import blosc2_ext + +pytestmark = pytest.mark.skipif( + not getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(), + reason="Installed runtime has no draft descriptor ABI", +) + + +def test_authoring_typed_snapshots(monkeypatch, tmp_path): + validate = blosc2.validate_portable_dsl + with monkeypatch.context() as guard: + guard.setattr( + blosc2_ext, "PortableArtifactHandle", lambda *a: pytest.fail("Artifact loader invoked") + ) + cache = tmp_path / "validation-jit-cache" + guard.setenv("CC", "/no/compiler") + guard.setenv("ME_DSL_JIT_COMPILER", "cc") + guard.setenv("ME_DSL_JIT", "1") + guard.setenv("ME_DSL_JIT_CACHE_DIR", str(cache)) + assert validate( + "# me:compiler=cc\ndef k(x):\n while 1:\n pass\n return x\n", + {"x": "float32"}, + "float32", + language_version="1.0", + )["valid"] + assert validate( + "# me:compiler=cc\ndef k(x, y):\n return sin(x) + y\n", + {"y": "int16", "x": "float32"}, + "float32", + language_version="1.0", + )["valid"] + assert validate("def k(x):\n return upper(x)\n", {"x": "U4"}, "U4", language_version="1.0")[ + "valid" + ] + assert validate("def k(x):\n return x\n", {"x": ">S4"}, "S4", language_version="1.0")["valid"] + assert not validate("def k(x):\n return upper(x)\n", {"x": "U4"}, "U3", language_version="1.0")[ + "valid" + ] + scalar = "def k(x):\n return sum(x)\n" + assert validate(scalar, {"x": "int16"}, "int64", language_version="1.0", cardinality="block_scalar")[ + "valid" + ] + assert ( + validate(scalar, {"x": "int16"}, "int64", language_version="1.0", cardinality="elementwise")[ + "status" + ] + == "invalid_signature" + ) + nd = "def k(x):\n return x + _i1\n" + missing = validate(nd, {"x": "int64"}, "int64", language_version="1.0") + assert not missing["valid"] + assert "rank" in missing["error"] + assert validate(nd, {"x": "int64"}, "int64", language_version="1.0", ndim=2)["valid"] + pair = {"x": "int64", "y": "uint64"} + assert validate("def k(x, y):\n return x < y\n", pair, "bool", language_version="1.0")["valid"] + assert not validate("def k(x, y):\n return x + y\n", pair, "float64", language_version="1.0")[ + "valid" + ] + for body in ("print(x)\n return x", "return callback(x)", "return x & 1"): + rejected = validate( + f"def k(x):\n {body}\n", {"x": "float32"}, "float32", language_version="1.0" + ) + assert not rejected["valid"] + assert rejected["line"] == 2 + assert rejected["column"] > 0 + invalid = validate("def k(x):\n return (\n", {"x": "float32"}, "float32", language_version="1.0") + assert invalid["status"] == "invalid_source" + assert invalid["line"] > 0 + assert invalid["column"] > 0 + assert not cache.exists() + + offset = np.uint64(7) + + @blosc2.dsl_kernel + def author(x): + return x + offset + + record = author.export({"x": "uint64"}, "uint64", version="1.0") + offset = np.uint64(99) + kernel = blosc2.PortableKernel.from_json(record) + np.testing.assert_array_equal(kernel.evaluate({"x": np.arange(3, dtype="uint64")}), [7, 8, 9]) + text = "ß" + + @blosc2.dsl_kernel + def string_author(x): + return x + text + + record = string_author.export({"x": "U2"}, "U3", version="1.0") + text = "changed" + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(record).evaluate({"x": np.array(["a"], dtype="U2")}), ["aß"] + ) + static = blosc2.DSLKernel.from_source("def rows(x):\n return sum(x) + _n0\n") + record = static.export({"x": "int64"}, "int64", version="1.0", cardinality="block_scalar", ndim=1) + assert blosc2.PortableKernel.from_json(record).result_cardinality == "block_scalar" + + minimum = np.int64(np.iinfo(np.int64).min) + + @blosc2.dsl_kernel + def minimum_capture(x): + return x + minimum + + record = minimum_capture.export( + {"x": "float64"}, "float64", version="1.0", capture_dtypes={"minimum": "float64"} + ) + minimum = np.int64(0) + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(record).evaluate({"x": np.array([0.0])}), [-float(2**63)] + ) + loop = blosc2.DSLKernel.from_source("def k(x):\n for i in range(x):\n y = i\n return i\n") + native_loop = blosc2.PortableKernel.from_json(loop.export({"x": "int64"}, "int64", version="1.0")) + np.testing.assert_array_equal(native_loop.evaluate({"x": np.array([2, 4], dtype="int64")}), [1, 3]) + + @blosc2.dsl_kernel + def static_row(row): + return np.sqrt(row["x"]) + row["y"] + + bound = dict(zip(static_row.input_names, ("float64", "float64"), strict=True)) + native = blosc2.PortableKernel.from_json(static_row.export(bound, "float64", version="1.0")) + arrays = dict(zip(static_row.input_names, (np.array([4.0]), np.array([3.0])), strict=True)) + np.testing.assert_array_equal(native.evaluate(arrays), [5.0]) + with pytest.raises(blosc2.PortableArtifactError): + author.export({"x": "uint64"}, "uint64", version="1.0", constants={"unknown": 1}) + + +def test_table_native_row_contract_roundtrip(tmp_path, monkeypatch): + @dataclass + class Row: + x: int = 0 + + source = blosc2.DSLKernel.from_source("def rows(x):\n return sum(x) + _flat_idx + _n0\n") + kernel = blosc2.PortableKernel.from_json(source.export({"x": "int64"}, "int64", version="1.0", ndim=1)) + table = blosc2.CTable(Row, new_data={"x": [2, 4]}, create_summary_index=False) + table.add_computed_column("virtual", kernel, inputs={"x": "x"}) + old_draft = {**table._computed_cols["virtual"], "row_domain": "independent"} + assert table._validate_portable_table_metadata(old_draft)["kind"] == "portable" + with pytest.raises(ValueError, match="independent"): + table._validate_portable_table_metadata({**old_draft, "row_domain": "dynamic"}) + table.add_generated_column("stored", values=source, dtype="int64", inputs={"x": "x"}) + np.testing.assert_array_equal(table["virtual"][:], [3, 5]) + table.append({"x": 8}) + np.testing.assert_array_equal(table["stored"][:], [3, 5, 9]) + table.refresh_generated_column("stored") + path = tmp_path / "table.b2d" + table.save(str(path)) + import blosc2.b2objects as b2objects + + monkeypatch.setattr( + b2objects, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + # The public package attribute is a decorator; import the actual module. + dsl_module = importlib.import_module("blosc2.dsl_kernel") + monkeypatch.setattr( + dsl_module, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + reopened = blosc2.CTable.open(str(path), mode="a") + np.testing.assert_array_equal(reopened["virtual"][:], [3, 5, 9]) + reopened.append({"x": 10}) + np.testing.assert_array_equal(reopened["stored"][:], [3, 5, 9, 11]) + framed = blosc2.ctable_from_cframe(reopened.to_cframe()) + np.testing.assert_array_equal(framed["stored"][:], [3, 5, 9, 11]) + reopened.materialize_computed_column("virtual", new_name="derived") + reopened.delete(1) + copied = reopened.copy(compact=True) + np.testing.assert_array_equal(copied["virtual"][:], [3, 9, 11]) + np.testing.assert_array_equal(copied["derived"][:], [3, 9, 11]) + copied.extend({"x": [12, 14]}) + np.testing.assert_array_equal(copied["stored"][:], [3, 9, 11, 13, 15]) + constructor = blosc2.DSLKernel.from_source("def constant_row():\n return _n0\n") + constructed = blosc2.PortableKernel.from_json(constructor.export({}, "int64", version="1.0", ndim=1)) + copied.add_generated_column("constant", values=constructed, inputs={}) + copied.append({"x": 16}) + copied.refresh_generated_column("constant") + np.testing.assert_array_equal(copied["constant"][:], np.ones(6, dtype="int64")) + sentinel = tmp_path / "preserved.b2z" + sentinel.write_bytes(b"original destination") + copied._computed_cols["virtual"]["artifact"] = "{}" + with pytest.raises(blosc2.PortableArtifactError): + copied.save(str(sentinel)) + assert sentinel.read_bytes() == b"original destination" + with pytest.raises(TypeError, match="row_domain"): + table.add_computed_column("invalid", kernel, inputs={"x": "x"}, row_domain="dynamic") + assert not hasattr(table, "add_portable_computed_column") + assert not hasattr(table, "add_portable_generated_column") + + @dataclass + class TextRow: + label: str = blosc2.field(blosc2.string(max_length=4), default="") + + text_table = blosc2.CTable(TextRow, new_data={"label": ["ß a", "ab"]}, create_summary_index=False) + upper = blosc2.DSLKernel.from_source("def upper_row(x):\n return upper(x)\n") + text_table.add_generated_column("upper", values=upper, dtype="U4", inputs={"x": "label"}) + text_table.append({"label": "z"}) + np.testing.assert_array_equal(text_table["upper"][:], ["SS A", "AB", "Z"]) + + @dataclass + class VectorRow: + vector: object = blosc2.field(blosc2.ndarray((3,), dtype=blosc2.int64())) + bias: int = 0 + + vector_table = blosc2.CTable( + VectorRow, new_data={"vector": [[1, 2, 3], [4, 5, 6]], "bias": [1, 2]}, create_summary_index=False + ) + total = blosc2.DSLKernel.from_source("def total_row(x, bias):\n return sum(x + bias) + _n0\n") + vector_table.add_generated_column( + "total", + values=total, + dtype="int64", + cardinality="block_scalar", + inputs={"x": "vector", "bias": "bias"}, + ) + vector_table.append({"vector": [7, 8, 9], "bias": 3}) + vector_table.refresh_generated_column("total") + restored = blosc2.ctable_from_cframe(vector_table.to_cframe()) + np.testing.assert_array_equal(restored["total"][:], [12, 24, 36]) + + +@pytest.mark.parametrize("generated", [False, True]) +def test_unified_table_registration_is_row_local_and_validated(tmp_path, generated): + @dataclass + class Row: + amount: float = 0.0 + + table = blosc2.CTable( + Row, + urlpath=str(tmp_path / "rows.b2d"), + mode="w", + new_data={"amount": [2.0, 4.0, 8.0]}, + create_summary_index=False, + ) + kernel = blosc2.DSLKernel.from_source("def center(x):\n return x - mean(x)\n") + if generated: + table.add_generated_column("centered", values=kernel, inputs={"x": "amount"}, create_index=True) + else: + table.add_computed_column("centered", kernel, inputs={"x": "amount"}) + np.testing.assert_array_equal(table["centered"][:], [0, 0, 0]) + table.append({"amount": 16.0}) + np.testing.assert_array_equal(table["centered"][1:3], [0, 0]) + schema = table._schema_dict_with_computed() + recipe = schema["materialized_columns" if generated else "computed_columns"][0] + assert "row_domain" not in recipe + assert "dsl_source" not in recipe + invalid = blosc2.DSLKernel.from_source("def invalid(x):\n return callback(x)\n") + if generated: + with pytest.raises(blosc2.PortableArtifactError): + table.add_generated_column("invalid", values=invalid, inputs={"x": "amount"}) + else: + with pytest.raises(blosc2.PortableArtifactError): + table.add_computed_column("invalid", invalid, inputs={"x": "amount"}) + assert "invalid" not in table.col_names + table.close() + restored = blosc2.CTable.open(str(tmp_path / "rows.b2d")) + np.testing.assert_array_equal(restored["centered"][:], [0, 0, 0, 0]) + restored.close()