From 16053b40b2288131e052fd9e1b2f7133b7fb1402 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 13:37:34 +0200 Subject: [PATCH 01/30] Add native-source DSL kernels and portable conformance tests --- doc/reference/dsl_syntax.md | 26 ++ plans/portable-DSLs-first-version.md | 573 +++++++++++++++++++++++++++ src/blosc2/dsl_kernel.py | 61 ++- tests/test_dsl_portable.py | 89 +++++ 4 files changed, 748 insertions(+), 1 deletion(-) create mode 100644 plans/portable-DSLs-first-version.md create mode 100644 tests/test_dsl_portable.py diff --git a/doc/reference/dsl_syntax.md b/doc/reference/dsl_syntax.md index 2568d3171..5654903ff 100644 --- a/doc/reference/dsl_syntax.md +++ b/doc/reference/dsl_syntax.md @@ -18,6 +18,32 @@ def kernel(x, y): Use Python-style indentation and always return a value on the paths you execute. +### Native source without a Python function + +`blosc2.DSLKernel.from_source(source)` constructs a native-only kernel without +executing source as Python or inspecting Python globals: + +```python +source = """# me:compiler=tcc +def affine(x): + return x * 2.0 + 1.0 +""" +kernel = blosc2.DSLKernel.from_source(source) +result = blosc2.lazyudf(kernel, (x,), dtype="float64")[:] +``` + +The source and header pragmas are preserved verbatim. Input dtypes come from the +supplied arrays; pass an explicit output dtype. The native compiler validates the +body with these types during computation. The constructor inspects only the +plain positional function header and does not certify portable-profile support. +Frontend conveniences, captured globals, and Python-function fallback are not +available on this path; use native function spellings such as `sin(x)`. + +This is separate from the existing Python-based persistence reconstruction path. +Portable artifact export/import is still under development. The full native +language reference and experimental portable profile live in miniexpr's +`doc/dsl-syntax.md` and `doc/dsl-spec/0.1.md`, respectively. + `@blosc2.jit` auto-detects this DSL: a decorated function whose body contains an `if`/`for`/`while` and that compiles under this grammar is dispatched here automatically, so its branches and loops actually run, once per chunk, instead diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md new file mode 100644 index 000000000..4d33434da --- /dev/null +++ b/plans/portable-DSLs-first-version.md @@ -0,0 +1,573 @@ +# Portable DSL v0.1: first implementation plan + +**Status:** Implementation plan for the `portable-dsls` branch. Proposed API names +and wire-format details below remain experimental until the first implementation +and conformance tests establish them. + +### Implementation progress + +The first raw-source slice is implemented across Python-Blosc2 and `../miniexpr`: + +- Draft native profile: `../miniexpr/doc/dsl-spec/0.1.md`. +- Native-only Python constructor: `blosc2.DSLKernel.from_source(source)`, with no + Python `exec`, captures, frontend rewriting, or Python-function fallback. +- Shared finite-float64 fixtures and standalone C runner: + `../miniexpr/tests/portable-dsl/`, covering branches, chained comparisons, + bounded loops, and reordered native input bindings. +- Native compiler selection now honors explicit source pragmas ahead of the + `ME_DSL_JIT_COMPILER` environment default, with regression tests. +- Interpreter-only and bundled-TCC build configurations exercise the corpus; + required-JIT tests reject silent fallback. + +The reviewed native baseline is miniexpr revision +`a1f845b590582a85e5fac04184f3f4758e7ca9a5`. Python-Blosc2's existing dependency +pin is unchanged; its installed extension and the authoritative native checkout +are tested independently. Integrating the updated native dependency remains +necessary before claiming the precedence change in distributed Python builds. + +This is **not yet the full v0.1 completion gate**: the wider dtype/semantic audit, +portable-profile validator, artifact schema/loader, and captured-constant export +are still pending. The raw-source constructor accepts native DSL, not only the +draft portable subset. No new dependency or persisted artifact format has been +introduced. + +## 1. Goal and scope + +Establish a small, versioned miniexpr kernel language that executes independently +of Python, then demonstrate Python export and standalone C import of a portable +kernel artifact. Deliver this vertical slice before deciding how far to extend +the language, its packaging, or other language bindings. + +This implements the first stage of [the broader proposal](portable-DSLs.md), with +two refinements: + +- Native execution and cross-language conformance come before artifact design. +- Compiler pragmas remain in portable source and override local compiler defaults, + while retaining best-effort fallback. + +Python-like syntax is compatible with a language-independent standard. The +contract is a documented grammar and semantics, implemented without Python, not +the absence of syntax familiar to Python users. + +### Two independently useful deliverables + +1. **Portable source:** identical source plus an explicit signature can execute + through miniexpr's C API and Python-Blosc2. +2. **Portable artifact:** a small manifest carries that source, signature, typed + constants, and semantic requirements between environments. + +The first deliverable must not depend on completing the second. + +### Initial required scope + +- One named kernel, scalar-per-element execution, one scalar result per element. +- An explicit ordered input signature and output dtype. +- A documented subset of existing native numeric and Boolean syntax. +- Native interpreter execution as the mandatory baseline. +- TCC execution where supported, with tests proving which backend actually ran. +- Raw-source construction from Python without reconstructing a Python function. +- Python export of a kernel with a captured numeric scalar constant. +- A standalone C loader that consumes the exported artifact without Python-side + preparation at load time. +- Version checks, typed binding validation, and useful unsupported-feature errors. + +### Deferred beyond the initial completion gate + +- Complete string/bytes support and normalization of every authoring convenience. +- New method syntax, membership syntax, or split unpacking in miniexpr. +- Reduction kernels, index symbols, arbitrary rank constraints, and broadcasting. +- Complex, structured, object, datetime, nullable, and variable-length dtypes. +- External C functions, closures, and host callbacks in portable artifacts. +- `print` and other observable block-level side effects. +- Whole lazy computation graphs, array storage references, and remote access. +- Integration into Blosc2 persistence containers and generated-code caches. +- Mandatory JavaScript, Rust, or Julia implementations. + +Existing functionality outside this subset continues to work through existing +APIs; it simply is not certified by the initial portable profile. + +### Baseline after reviewing the updated miniexpr checkout + +The authoritative checkout is `../miniexpr`. Its existing `doc/dsl-syntax.md` +is already titled **miniexpr DSL Syntax (Canonical Reference)** and documents +native syntax and several exact semantic/error rules. `doc/dsl-usage.md` already +demonstrates standalone C compilation/evaluation and documents JIT controls. +This work standardizes a portable subset of that language; it does not introduce +native DSL execution from scratch or replace the existing reference. + +In particular, the full language already supports reductions, reserved ND symbols, +`print`, and string-valued kernels. Those features need explicit profile exclusions +in v0.1 rather than descriptions implying they are absent from miniexpr. + +## 2. Ownership and implementation boundaries + +### miniexpr owns the portable language + +Keep the normative specification, native validation, interpreter behavior, and +language conformance corpus with miniexpr. Its implementation is the initial +implementation of the specification, not an implicit substitute for one. + +Keep the kernel DSL distinct from miniexpr's classic expression API. Do not +silently expand or change that API when implementing portable kernel support. + +### Documentation layout and versioning + +Use these paths in the authoritative miniexpr repository: + +- `doc/dsl-syntax.md`: retain the current canonical reference for the full native + DSL. Extend or clarify it where the semantic audit finds omissions; preserve + its existing role and links from other documentation. +- `doc/dsl-usage.md`: retain the practical C API and runtime configuration guide. +- `doc/dsl-spec/0.1.md`: add the normative **portable profile** specification, + identifying the admitted grammar, dtype/function matrix, semantic requirements, + and explicit exclusions from the full DSL. +- `doc/dsl-spec/artifact-0.1.md`: add the versioned manifest and typed-binding + specification once the portable-source milestone works. + +During drafting, derive rules from the canonical reference rather than inventing +a second grammar. At publication, make the versioned profile's normative rules +self-contained or pin incorporated material to an immutable miniexpr revision. +A moving `doc/dsl-syntax.md` link must not silently change the meaning of a +published v0.1 artifact. Link from the current canonical reference to the profile +and explain full-language versus portable-profile support. + +Native work must be made in the authoritative miniexpr checkout and integrated +through the dependency mechanism used by Python-Blosc2. Build-tree copies under +`build*/_deps/` are inspection/build artifacts, not the source of record. + +### Python-Blosc2 owns authoring and integration + +Use `src/blosc2/dsl_kernel.py` as the integration point for current kernel +authoring, normalization, and capture handling. Inspect the existing extension +bridge before adding new native entry points. If artifact handling merits its +own module, use a non-underscored name such as `dsl_portable.py`. + +Separate the native specification from Python guidance in +`doc/reference/dsl_syntax.md`. Link to the normative miniexpr reference and clearly +identify decorator behavior and frontend-only conveniences. + +### Keep the layers independently testable + +1. Parse and validate native source against the portable profile. +2. Validate signature and typed bindings. +3. Check target semantic capabilities. +4. Compile or interpret using the selected execution preference. +5. Execute on caller-supplied buffers. + +Artifact decoding should feed these same operations. It should not introduce a +second language validator or execute frontend rewriting at import time. + +## 3. Define the initial language profile + +Use separate identifiers for the language version and artifact schema version. +Suggested initial values are `0.1` for each; these are experimental compatibility +identifiers, not miniexpr package versions. + +### Candidate core, to freeze after the native audit + +- One top-level `def`, positional named parameters, indentation-based blocks. +- Local assignments, supported compound assignments, and `return`. +- `if`/`elif`/`else`, `for ... in range(...)`, bounded test cases for `while`, + `break`, `continue`, and `pass`. +- Native numeric literals, arithmetic, comparisons, chained comparisons, and + Boolean operators. +- A short, explicit whitelist of native casts and mathematical functions. +- Comments, docstrings, semicolons, and multiline syntax already accepted natively. +- Initially target `bool`, `int32`, `int64`, `float32`, and `float64`; include only + types and operations whose behavior is specified and verified across the + interpreter and supported JIT paths. Do not claim unsigned or mixed-type support + merely because a host API accepts the dtype. + +Start the first executable fixture with `float64`; expand to the frozen core +before declaring v0.1 complete. Record the final operator/function/type matrix in +the normative specification rather than relying on this candidate list. + +### Semantic decisions required before freezing the profile + +Document and test: + +- Parameter binding, duplicate names, local assignment and type inference rules. +- Literal typing, promotions, return conversion, and explicit casts. +- Integer overflow, narrowing, signed division, remainder, and division by zero. +- Floating-point rounding assumptions, non-finite values, and signed zero. +- Evaluation order, single evaluation of chained operands, and short-circuiting. +- Boolean-result rules for `and`/`or`, without implying Python operand-return rules. +- Loop bounds, step handling, and errors such as a zero `range` step. +- Missing returns and use of uninitialized locals. +- The scalar-per-element execution model and independence from chunk boundaries. +- Error categories and when errors occur: validation, compilation, or execution. + +Carry forward the rules already documented in the native reference, verifying +them rather than treating them as undecided design choices: + +- Native parameters match variable names by set membership; caller variable order + may differ from source parameter order. +- Incompatible local reassignment and inconsistent return dtypes are compile-time + errors; an executed path without a return fails at runtime. +- A zero `range` step and exceeding the `while` iteration cap are runtime errors. + Specify the cap and its configuration/portability implications for this profile. +- `//=` lowers to `floor(a / b)`; do not assume Python integer floor-division + precision without auditing intermediate typing. +- Reserved ND names remain reserved even though index-aware kernels are excluded. +- The full DSL permits reductions at top level and in conditions, but not inside + `if`/`for`/`while` bodies. The v0.1 elementwise profile excludes reductions in + all positions, including conditions, to retain chunk-independent semantics. + +Audit existing behavior before promising a rule. If interpreter and JIT disagree, +either fix the discrepancy with regression coverage or exclude that operation/type +combination from v0.1. Do not standardize accidental C undefined behavior. + +Use a small core capability identifier rather than a capability for every token. +Add separate capabilities only for meaningful optional semantic features. Backend +names such as `tcc` are execution preferences, not language capabilities. + +## 4. Compiler preferences and interpreter baseline + +Miniexpr includes an interpreter. It also builds bundled libtcc by default on +supported targets, so ordinary users do not need a separately installed compiler. +TCC may be disabled in custom builds or unavailable on a target; Python-Blosc2 +currently disables it on Windows ARM64. + +For this implementation: + +1. Preserve `# me:compiler=tcc|cc` in portable source. +2. An explicit source pragma wins over the local compiler-selection default. +3. With no pragma, use the runtime's existing default selection policy. +4. Preserve existing best-effort fallback when the preferred backend is unavailable + or cannot compile the kernel. Use the interpreter when it satisfies the required + semantics. +5. Keep ordinary fallback quiet, while making the actual execution backend + observable to diagnostics and conformance tests. + +Audit the current selection path and codify its fallback order; do not invent a +new fallback chain as a side effect of artifact support. A deliberate local +interpreter-only execution mode remains useful for testing and deployments; it +is distinct from choosing a default compiler. + +Reuse the documented `ME_JIT_DEFAULT`, `ME_JIT_ON`, and `ME_JIT_OFF` policies, +including `me_eval_params.jit_mode`, rather than adding a parallel JIT toggle. +Test that explicit `ME_JIT_OFF` prevents JIT even with a compiler pragma: choosing +whether to JIT and choosing which compiler to prefer are separate controls. + +Floating-point requirements are independent of backend preferences. Define how +`# me:fp=strict|contract|fast` maps to the artifact's semantic requirements and +which interpreter/JIT modes satisfy them. Reject conflicting declarations rather +than silently allowing one to override the other. Do not promise identical bits +across all mathematical libraries or imply a stronger meaning of `strict` than +the specification defines. + +Compiler paths, cache directories, and diagnostic settings remain local. The +artifact contains a compiler preference, never instructions to locate or load an +arbitrary executable or library. + +## 5. Raw-source execution before packaging + +Build the first vertical slice around source such as: + +```text +# me:compiler=tcc +def affine(x, scale, offset): + y = x * scale + offset + if y < 0: + return 0.0 + return y +``` + +At this stage all three names can be ordinary explicitly typed inputs. Execute +the exact same source through a standalone C program and Python, using the same +input values and declared output type. + +Expose or reuse a Python raw-source construction path that accepts source and a +signature. Loading it must not use Python `exec`, fabricate a Python function, or +inspect globals. Final public names should follow the existing kernel API after +inspection; do not add competing constructors unnecessarily. + +The C executable must link miniexpr without libpython. Python may launch it as a +test subprocess, but the executable must also run directly from a shell with +prepared fixtures. Native execution requires no Python-generated temporary code. + +Start from the existing `me_compile()` / `me_eval()` / `me_free()` flow documented +in `doc/dsl-usage.md`; use `me_compile_nd_jit()` where its explicit JIT controls +are needed. Audit existing parser diagnostics and backend test helpers before +introducing any new native API. The new work here is a reusable conformance runner +and profile validation, not a replacement execution engine. + +## 6. Minimal artifact design + +After raw-source conformance works, use a standalone UTF-8 JSON manifest as the +first interchange format. Keep it small enough for a native loader and independent +of Blosc2 storage. Reassess container integration after v0.1 works end to end. + +### Proposed fields + +| Field | Meaning | +| --- | --- | +| `schema_version` | Artifact encoding/structure version | +| `language` | Language identifier and version | +| `requires` | Required semantic capability identifiers | +| `source` | Complete canonical native DSL source, including retained pragmas | +| `entry_point` | Name of the single kernel in the source | +| `inputs` | Ordered runtime input names and language-neutral dtypes | +| `constants` | Explicit named, typed scalar bindings | +| `output` | Output dtype and scalar-per-element execution contract | +| `semantics` | Required floating-point profile and other supported requirements | +| `metadata` | Optional non-semantic producer information | + +Freeze required fields and unknown-field handling in a small schema document. +Reject duplicate JSON keys, duplicate bindings, and unknown semantic fields. +Permit explicitly designated informational metadata without interpreting it. +Do not introduce a runtime schema-validation dependency just for this manifest. + +### Signature and constants + +Use the source's parameter list as the full native parameter order. Every +parameter must be supplied exactly once by either `inputs` or `constants`. +Disallow overlapping names, missing bindings, and unused binding entries. +`inputs` follows source order with constant parameters omitted. + +This ordering is an artifact convention, not a new native-language restriction. +The adapter must map names to the `me_variable` array and supply evaluation +pointers in that array's order, as the current native API requires. Include a +reordered-variable fixture to prevent accidental positional-only binding. + +For captured constants, export a collision-free explicit parameter and record its +typed value in `constants`. This avoids reliance on implicit native globals or +Python literal formatting. Reuse native scalar binding support if available; +otherwise provide an adapter with defined broadcast and lifetime behavior rather +than exposing an array-sized copy requirement in the artifact contract. + +### Typed scalar encoding + +- Boolean: JSON Boolean with dtype `bool`. +- Integer: decimal string with explicit fixed-width dtype and range validation. +- Float: exact IEEE-754 bit-pattern hex string with explicit `float32` or + `float64` dtype and fixed digit count, written most-significant byte first. +- Decode floats into the host representation explicitly; do not rely on native + endianness or JSON numeric precision. + +This preserves large integers, signed zero, infinities, and NaNs in transport. +Transporting a NaN bit pattern does not promise that arithmetic preserves its +payload. Strings/bytes get a separate encoding decision when that profile is added. + +### Compatibility + +For the experimental implementation, accept only explicitly supported language +and schema versions. Reject unknown versions clearly instead of guessing forward +compatibility. Keep frozen example artifacts as regression fixtures. Once a v0.1 +contract is published, change semantics under a new version rather than silently +reinterpreting existing artifacts. + +## 7. Python export and import + +### Export pipeline + +1. Obtain the existing author's source and explicit specialization signature. +2. Normalize supported frontend conveniences using the existing frontend machinery. +3. Discover scalar captures without calling the kernel or arbitrary user callbacks. +4. Resolve each supported capture to an explicit dtype and snapshot its value at + export time. Require an explicit type when inference would be ambiguous. +5. Replace captures with collision-free bound parameters. +6. Run native portable-profile validation on the resulting source and signature. +7. Derive required capabilities and emit the manifest deterministically. + +Do not treat `kernel.dsl_source` alone as the finished artifact. Audit whether +existing normalization or scalar specialization has already discarded type or +capture information; retain that information before lowering if necessary. + +The first required example captures a numeric scale constant. After export, +changing the Python global must not change execution of the saved artifact. +Reject unresolved names, arrays/objects as captures, external functions, and +unsupported constructs with the relevant name and source location where possible. + +### Import pipeline + +Decode and validate the artifact, construct the typed bindings, then use the same +native source path as C. Import must not require the original module, function, +decorators, globals, or frontend rewriters. + +Provide a raw-source portable validator first. An optional strict authoring mode +that rejects frontend sugar and implicit captures can build on it later; it need +not block the first export/import milestone. + +## 8. Native artifact loader + +Implement the loader as a thin adapter adjacent to miniexpr's native API, with +clear ownership of decoded source, binding buffers, compiled handles, and errors. +The language compiler must remain usable without JSON artifact support. + +Before choosing JSON machinery, inspect existing dependency policy and available +parsers. Reuse an appropriate existing facility where possible; any new dependency +requires an explicit decision. Avoid a hand-written partial JSON parser that only +accepts the Python exporter's preferred formatting. + +Loader stages: + +1. Decode JSON with bounded lengths and validate field structure. +2. Check schema/language versions and required capabilities. +3. Decode and range-check typed constants and signature information. +4. Validate entry point and exact binding coverage against native source. +5. Check semantic requirements independently of the compiler preference. +6. Construct an executable handle using the existing interpreter/JIT machinery. +7. Bind caller-provided data and execute; free all owned resources on every path. + +Define logical buffers as same-length inputs and output for the initial profile, +with scalar constants broadcast to each element. Specify empty-input behavior, +host dtype conversion, buffer ownership, and binding lifetime. Keep physical +strides, endian conversion, and storage layout in adapters rather than in source. + +Distinguish malformed artifacts, unsupported requirements, invalid source, +binding errors, and evaluation errors. Lack of the preferred compiler alone is +not an invalid artifact. + +The loader does not provide a sandbox. It must not execute Python, follow artifact +paths to load libraries, or use artifact data as shell commands. General resource +isolation remains a host responsibility rather than a new v0.1 subsystem. + +## 9. Conformance corpus and verification + +Create language-neutral fixtures containing source, signature, inputs, expected +outputs or error category, and numeric comparison rules. Keep native language +fixtures authoritative in miniexpr and consume the same cases from Python tests; +avoid independently maintained copies with diverging expectations. + +Seed the corpus from existing native tests, especially `test_dsl_syntax.c`, +`test_dsl_python_syntax.c`, `test_dsl_literals.c`, `test_dsl_comparisons.c`, +`test_dsl_bool_locals.c`, `test_dsl_guards.c`, and `test_dsl_jit_options.c`. +Reuse their helpers where appropriate and preserve implementation-level coverage; +only promote profile-relevant cases into the shared language-neutral corpus. + +Required groups: + +- Basic arithmetic, branches, loops, casts, and every admitted dtype/function. +- Chained comparisons, short-circuit guards, and operand evaluation order/count. +- Integer boundaries, promotions, narrowing, and defined arithmetic errors. +- Floating-point boundaries, signed zero, NaNs, infinities, and profile-specific + tolerances for mathematical functions. +- Signature ordering, captured constants, name collisions, and empty buffers. +- Invalid syntax, missing returns/bindings, unsupported types and capabilities. +- Malformed JSON, duplicate fields, invalid scalar encodings, unknown versions, + and inconsistent semantic declarations. +- Compiler pragma precedence, no-pragma defaults, and interpreter fallback. +- Frozen artifact import without its originating Python module. + +Use expected values derived from the specification, not only agreement between +two implementations. For single-evaluation tests, use native test instrumentation +where pure numeric outputs cannot reveal repeated evaluation; do not require +portable user callbacks just to make the tests observable. + +Run all core cases explicitly through the interpreter. Run the applicable corpus +through TCC, and through CC where available. JIT tests must inspect the actual +backend or use an existing no-fallback test mode so an interpreter fallback cannot +masquerade as JIT conformance. Fallback itself has separate tests. + +Include at least one configuration with TCC disabled and one normal bundled-TCC +configuration. Exercise representative supported platforms in CI as available. +JavaScript is a later capability-specific target, not a first-version gate. + +Python, build, and test commands in this repository must use the `blosc2` conda +environment. Run focused native tests and Python DSL tests as each layer changes, +then the repository's required checks before completion. Keep unrelated tests and +benchmarks outside the implementation loop unless failures justify investigation. + +## 10. Ordered work packages and completion gates + +### P0 — Audit and freeze a small contract + +- Locate authoritative miniexpr sources and dependency integration points. +- Record the reviewed miniexpr revision and use its existing canonical reference, + usage guide, and native DSL tests as the specification/conformance baseline. +- Inventory native parsing, dtype rules, execution APIs, capture specialization, + scalar bindings, and backend selection. +- Write the initial normative grammar/semantics and supported operation matrix. +- Record exclusions and resolve interpreter/JIT discrepancies for admitted cases. + +**Gate:** a precise candidate profile and first fixtures, with no dependence on +Python behavior to explain native semantics. + +### P1 — Execute identical raw source in C and Python + +- Add the standalone C runner and Python raw-source integration. +- Execute the affine example and control-flow cases with explicit signatures. +- Add interpreter/TCC backend observability and selection-policy tests. + +**Gate:** the same source executes without Python preparation in C and through +Python, and interpreter execution is verified explicitly. + +### P2 — Establish the shared core conformance suite + +- Expand fixtures across the frozen core syntax and dtype matrix. +- Document numerical comparison rules and runtime error categories. +- Verify interpreter/TCC parity and applicable CC coverage. + +**Gate:** all claimed v0.1 core semantics have representative positive, boundary, +and rejection coverage. The portable-source milestone is independently usable. + +### P3 — Implement the minimal artifact and C loader + +- Freeze the initial manifest and scalar encodings. +- Implement native decoding, binding validation, ownership, and error handling. +- Load a hand-authored artifact and reuse the P1 execution path. + +**Gate:** a standalone C executable loads and runs a fixture artifact without +Python and rejects malformed or unsupported artifacts explicitly. + +### P4 — Export and re-import a Python-authored kernel + +- Implement explicit-signature export and captured scalar parameterization. +- Implement Python import through the native portable-source path. +- Export in Python, execute in C, and compare against specified expected results. +- Verify snapshot semantics, deterministic export, and independence from the + original Python module. + +**Gate:** the complete Python-to-C artifact round trip works for the numeric core. + +### P5 — Document, stabilize, and review the first version + +- Publish the native reference and short Python/C examples. +- Document pragma precedence, fallback, version handling, and supported dtypes. +- Preserve frozen artifacts and complete focused regression/platform checks. +- Record practical limitations and review the next priorities with the user. + +**Gate:** v0.1 meets the checklist below. Further expansion is a separate decision. + +## 11. Follow-up candidates after v0.1 + +Extend the same corpus and manifest rather than redesigning the architecture. + +1. **Strings and bytes:** define encodings, lengths, case behavior, substring rules, + output storage, and dtype descriptions before claiming portable-profile support. + Start from existing `doc/strings.md` and `doc/dsl-syntax.md`: native string + locals and returns already work, widths have documented constraints, shorter + returns are NUL-padded, and string kernels execute on the interpreter rather + than the JIT. A string profile must not require TCC support to be portable. +2. **Native string methods:** consider a fixed whitelist lowering to existing + functions, without general object dispatch. +3. **Native membership:** consider `in`/`not in` for explicitly supported + string/bytes operands, with defined evaluation order. +4. **Frontend adapters:** NumPy aliases and pandas row access remain authoring and + binding conveniences; add portable column mappings when needed. +5. **Split unpacking:** decide missing-separator, empty-separator, result-count, + and evaluation-count semantics before lowering to `split_part` calls. Two + extraction calls do not automatically reproduce Python unpacking behavior. +6. **Additional execution profiles:** reductions, index-aware kernels, other + backends, and foreign-language bindings, each with matching conformance cases. + +Promotion of useful syntax into miniexpr is welcome when it benefits authors in +multiple environments. Eliminating a Python rewrite alone is not a sufficient +reason to enlarge the language. + +## 12. First-version acceptance checklist + +- [ ] Native versioned specification defines the admitted language subset. +- [ ] Same raw source and signature execute from C and Python. +- [ ] Interpreter passes the full core conformance corpus. +- [ ] Supported JIT paths pass applicable cases with actual backend verification. +- [ ] Compiler pragmas survive export and override local compiler defaults. +- [ ] Best-effort fallback preserves required semantics and remains quiet by default. +- [ ] Explicit typed constants survive a Python export/C import round trip. +- [ ] Saved artifacts execute without the original Python module or Python runtime. +- [ ] Loaders reject unsupported versions/capabilities and malformed bindings. +- [ ] Dtype, numeric, error, and buffer contracts are documented and tested. +- [ ] Existing Python authoring and evaluation APIs retain their behavior. +- [ ] Frozen artifacts and shared fixtures guard the published v0.1 contract. +- [ ] Follow-up scope is reviewed after this slice is working, not assumed upfront. diff --git a/src/blosc2/dsl_kernel.py b/src/blosc2/dsl_kernel.py index b19f1965a..d679c5d43 100644 --- a/src/blosc2/dsl_kernel.py +++ b/src/blosc2/dsl_kernel.py @@ -689,7 +689,64 @@ def _cmpop(self, op: ast.cmpop): class DSLKernel: - """Wrap a Python function and optionally extract a miniexpr DSL kernel from it.""" + """Wrap a Python function, or construct a native-only kernel with ``from_source``.""" + + @classmethod + def from_source(cls, source: str) -> DSLKernel: + """Construct a native-only kernel without executing Python source. + + Preserve the source, including header pragmas, verbatim. Only the function + header is inspected here; miniexpr validates the body when compiling with + the input and output dtypes supplied to :func:`blosc2.lazyudf`. + No frontend rewriting, captured globals, or Python fallback is provided. + This accepts native DSL source, not a portable-profile certification. + """ + if not isinstance(source, str): + raise TypeError("DSL source must be a string") + if "\x00" in source: + raise ValueError("Native DSL source must not contain NUL characters") + try: + tokens = list(tokenize.generate_tokens(StringIO(source).readline)) + first = next(t for t in tokens if t.type not in (tokenize.COMMENT, tokenize.NL)) + if first.type != tokenize.NAME or first.string != "def": + raise ValueError("DSL source must start with a function definition") + _, _, colon = _find_def_signature_span(source) + if colon is None: + raise ValueError("DSL source requires a function header") + lines = source.splitlines(keepends=True) + header = "".join(lines[: colon.end[0] - 1]) + lines[colon.end[0] - 1][: colon.end[1]] + function = ast.parse(header + "\n pass\n").body[0] + args = function.args + if ( + args.posonlyargs + or args.vararg + or args.kwarg + or args.kwonlyargs + or args.defaults + or args.kw_defaults + or function.returns + or any(arg.annotation for arg in args.args) + ): + raise ValueError("Native DSL parameters must be plain positional names") + names = [arg.arg for arg in args.args] + if len(set(names)) != len(names): + raise ValueError("Native DSL parameter names must be unique") + except (SyntaxError, tokenize.TokenError, IndentationError, StopIteration) as error: + raise ValueError(f"Invalid native DSL function header: {error}") from error + + kernel = cls.__new__(cls) + kernel.func = None + kernel.__name__ = kernel.__qualname__ = function.name + kernel.__doc__ = None + kernel._sig = None + kernel._sig_has_varargs = False + kernel._sig_npositional = len(names) + kernel._legacy_udf_signature = False + kernel.dsl_source = source + kernel.input_names = names + kernel.dsl_error = None + kernel.row_param = kernel.row_columns = None + return kernel def __init__(self, func): self.func = func @@ -873,6 +930,8 @@ def _input_names_from_signature(func_node: ast.FunctionDef) -> list[str]: return [a.arg for a in (args.posonlyargs + args.args)] def __call__(self, inputs_tuple, output, offset=None): + if self.func is None: + raise RuntimeError("A native-source DSL kernel cannot execute as a Python function") if self.dsl_error is not None: raise self.dsl_error if self._legacy_udf_signature: diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py new file mode 100644 index 000000000..29f0acffd --- /dev/null +++ b/tests/test_dsl_portable.py @@ -0,0 +1,89 @@ +"""Initial raw-source portable conformance; the corpus is owned by miniexpr.""" + +import os +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +import blosc2 +from blosc2.dsl_kernel import DSLKernel + +CORPUS = Path( + os.environ.get( + "MINIEXPR_PORTABLE_CORPUS", + Path(__file__).resolve().parents[2] / "miniexpr" / "tests" / "portable-dsl", + ) +) + + +@pytest.mark.parametrize("case", ["affine", "chains", "loops"]) +@pytest.mark.parametrize("jit", [False, True]) +def test_native_source_corpus(case, jit): + if not CORPUS.is_dir(): + pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") + source = (CORPUS / f"{case}.dsl").read_text() + words = (CORPUS / f"{case}.txt").read_text().split() + count, nvars = map(int, words[:2]) + names = words[2 : 2 + nvars] + rows = np.array(words[2 + nvars :], dtype=np.float64).reshape(count, nvars + 1) + kernel = DSLKernel.from_source(source) + by_name = dict(zip(names, rows[:, :-1].T, strict=True)) + inputs = [by_name[name].copy() for name in kernel.input_names] + result = blosc2.lazyudf(kernel, inputs, dtype=np.float64, jit=jit)[:] + np.testing.assert_allclose(result, rows[:, -1], rtol=1e-12, atol=1e-12) + assert kernel.dsl_source == source + assert kernel.func is None + + runner = os.environ.get("MINIEXPR_PORTABLE_RUNNER") + if runner: + completed = subprocess.run( + [runner, str(CORPUS / f"{case}.dsl"), str(CORPUS / f"{case}.txt"), "on" if jit else "off"], + check=True, + capture_output=True, + text=True, + ) + lines = completed.stdout.splitlines() + assert lines[0] == f"jit={int(jit)}" + np.testing.assert_allclose(np.array(lines[1:], dtype=float), rows[:, -1], rtol=1e-12, atol=1e-12) + + +def test_from_source_never_executes_python(tmp_path): + target = tmp_path / "executed" + with pytest.raises(ValueError, match="start with a function"): + DSLKernel.from_source(f"open({str(target)!r}, 'w')\ndef kernel(x):\n return x\n") + with pytest.raises(ValueError, match="plain positional"): + DSLKernel.from_source(f"def kernel(x=open({str(target)!r}, 'w')):\n return x\n") + assert not target.exists() + + +@pytest.mark.parametrize( + "source", ["", "def broken(", "def k(x, x):\n return x", "def k(x):\n return x\x00"] +) +def test_from_source_invalid_header(source): + with pytest.raises(ValueError): + DSLKernel.from_source(source) + + +def test_from_source_native_body_and_no_python_fallback(): + source = "# me:compiler=tcc\ndef kernel(x):\n return !x\n" + kernel = DSLKernel.from_source(source) + assert kernel.dsl_source == source + assert kernel.input_names == ["x"] + with pytest.raises(RuntimeError, match="cannot execute as a Python"): + kernel((np.array([1.0]),), np.empty(1)) + + +@pytest.mark.parametrize("jit", [False, True]) +def test_from_source_without_external_corpus(jit): + source = "# me:compiler=tcc\ndef kernel(x):\n return !x\n" + kernel = blosc2.DSLKernel.from_source(source) + result = blosc2.lazyudf(kernel, (np.array([0.0, 1.0, -1.0]),), dtype=np.bool_, jit=jit)[:] + np.testing.assert_array_equal(result, [True, False, False]) + + +def test_from_source_invalid_native_body(): + kernel = DSLKernel.from_source("def kernel(x):\n return x[0]\n") + with pytest.raises((NotImplementedError, RuntimeError)): + blosc2.lazyudf(kernel, (np.arange(4, dtype=np.float64),), dtype=np.float64)[:] From 4321149c0baea44cfeff472501d0100a65cae052 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 14:08:14 +0200 Subject: [PATCH 02/30] Expand typed portable DSL conformance coverage --- plans/portable-DSLs-first-version.md | 14 +++++ tests/test_dsl_portable.py | 83 ++++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 11 deletions(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 4d33434da..3bc2ba17d 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -31,6 +31,20 @@ are still pending. The raw-source constructor accepts native DSL, not only the draft portable subset. No new dependency or persisted artifact format has been introduced. +The next conformance slice expands the shared corpus with explicit +input/output dtypes and expected compile/evaluation outcomes. It covers all five +candidate dtypes, exact int64 boundaries and comparisons above `2**53`, signed +zero and non-finite identity, Boolean-result semantics, bounded loop control, +small finite casts, and sampled `sin` results. Integer fixtures are decoded and +compared without a floating-point intermediate. CTest also generates CC variants +that change only the compiler preference, uses build-local JIT caches, and applies +timeouts. Missing-return kernels now have runtime JIT support: successful paths, +mixed successful/failing elements, loop returns, and hybrid vector cleanup are +tested under required-JIT policies. Semantic missing-return errors propagate +without interpreter retry; generic backend failures retain existing fallback. +This advances P2 coverage but does not freeze +overflow, general promotions/conversions, or global transcendental accuracy rules. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 29f0acffd..519172a53 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -18,35 +18,96 @@ ) -@pytest.mark.parametrize("case", ["affine", "chains", "loops"]) +CASES = [ + "affine", + "chains", + "loops", + "int32", + "int64", + "float32", + "bool", + "compare", + "special_floats", + "flow", + "boolean_results", + "cast_int", + "math", + "unknown_name", + "invalid_index", + "zero_step", + "missing_return", + "missing_return_ok", + "missing_return_mixed", + "missing_return_loop", + "missing_return_loop_ok", + "missing_return_hybrid", + "missing_return_hybrid_ok", +] + + +def typed_values(values, dtype): + # NumPy treats nonempty strings as True; fixture Boolean tokens are 0/1. + return np.array([int(value) for value in values] if dtype == np.dtype("bool") else values, dtype=dtype) + + +def assert_typed_result(actual, expected): + assert actual.dtype == expected.dtype + if expected.dtype.kind in "biu": + np.testing.assert_array_equal(actual, expected) + else: + tolerance = 1e-6 if expected.dtype == np.dtype("float32") else 1e-12 + np.testing.assert_allclose(actual, expected, rtol=tolerance, atol=tolerance, equal_nan=True) + zeros = expected == 0 + np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) + + +@pytest.mark.parametrize("case", CASES) @pytest.mark.parametrize("jit", [False, True]) def test_native_source_corpus(case, jit): if not CORPUS.is_dir(): pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") source = (CORPUS / f"{case}.dsl").read_text() words = (CORPUS / f"{case}.txt").read_text().split() - count, nvars = map(int, words[:2]) - names = words[2 : 2 + nvars] - rows = np.array(words[2 + nvars :], dtype=np.float64).reshape(count, nvars + 1) + outcome, input_type, output_type = words[:3] + count, nvars = map(int, words[3:5]) + names = words[5 : 5 + nvars] + rows = np.array(words[5 + nvars :]).reshape(count, nvars + 1) + input_dtype, output_dtype = np.dtype(input_type), np.dtype(output_type) + expected = typed_values(rows[:, -1], output_dtype) kernel = DSLKernel.from_source(source) - by_name = dict(zip(names, rows[:, :-1].T, strict=True)) - inputs = [by_name[name].copy() for name in kernel.input_names] - result = blosc2.lazyudf(kernel, inputs, dtype=np.float64, jit=jit)[:] - np.testing.assert_allclose(result, rows[:, -1], rtol=1e-12, atol=1e-12) + by_name = {name: typed_values(rows[:, index], input_dtype) for index, name in enumerate(names)} + inputs = [by_name[name] for name in kernel.input_names] + if outcome == "ok": + result = blosc2.lazyudf(kernel, inputs, dtype=output_dtype, jit=jit)[:] + assert_typed_result(result, expected) + else: + with pytest.raises((NotImplementedError, RuntimeError)): + blosc2.lazyudf(kernel, inputs, dtype=output_dtype, jit=jit)[:] assert kernel.dsl_source == source assert kernel.func is None runner = os.environ.get("MINIEXPR_PORTABLE_RUNNER") if runner: completed = subprocess.run( - [runner, str(CORPUS / f"{case}.dsl"), str(CORPUS / f"{case}.txt"), "on" if jit else "off"], + [ + runner, + str(CORPUS / f"{case}.dsl"), + str(CORPUS / f"{case}.txt"), + "on" if jit else "off", + ], check=True, capture_output=True, text=True, ) lines = completed.stdout.splitlines() - assert lines[0] == f"jit={int(jit)}" - np.testing.assert_allclose(np.array(lines[1:], dtype=float), rows[:, -1], rtol=1e-12, atol=1e-12) + if outcome == "compile_error": + assert lines == [outcome] + else: + assert lines[0] == f"jit={int(jit)}" + if outcome == "eval_error": + assert lines[1:] == [outcome] + else: + assert_typed_result(typed_values(lines[1:], output_dtype), expected) def test_from_source_never_executes_python(tmp_path): From a5a128d44565d1b36e534ce25931ccd89fe5d3d1 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 14:29:10 +0200 Subject: [PATCH 03/30] Expose native portable DSL validation --- doc/reference/dsl_syntax.md | 22 +++++++ plans/portable-DSLs-first-version.md | 19 +++++- src/blosc2/__init__.py | 10 ++- src/blosc2/blosc2_ext.pyx | 74 ++++++++++++++++++++++ src/blosc2/dsl_kernel.py | 37 +++++++++++ tests/test_dsl_portable.py | 95 ++++++++++++++++++++++++++++ 6 files changed, 255 insertions(+), 2 deletions(-) diff --git a/doc/reference/dsl_syntax.md b/doc/reference/dsl_syntax.md index 5654903ff..6dd9dc375 100644 --- a/doc/reference/dsl_syntax.md +++ b/doc/reference/dsl_syntax.md @@ -44,6 +44,28 @@ Portable artifact export/import is still under development. The full native language reference and experimental portable profile live in miniexpr's `doc/dsl-syntax.md` and `doc/dsl-spec/0.1.md`, respectively. +### Experimental portable-profile validation + +```python +info = blosc2.validate_portable_dsl(source, {"x": "float64"}, "float64") +if not info["valid"]: + print(info["status"], info["line"], info["column"], info["error"]) +``` + +This native-only check uses raw source and explicit logical dtypes, with no +frontend rewriting or captured globals. It neither executes the kernel nor +invokes a JIT compiler. It reports version, signature, source, and unsupported +feature errors separately. Potential runtime errors, including missing returns +on some paths, do not automatically invalidate a program. + +The draft profile admits five numeric/Boolean dtypes with homogeneous input types +and a small function/operator set; it is narrower than the full language described +below. Validation is not a sandbox, backend-support probe, or a guarantee about +arbitrary runtime data. Older native builds report `runtime_unsupported` rather +than attempting Python validation. Native dependency integration for this +experimental API is pending; development builds can use the local miniexpr source +via CMake's `FETCHCONTENT_SOURCE_DIR_MINIEXPR` override. + `@blosc2.jit` auto-detects this DSL: a decorated function whose body contains an `if`/`for`/`while` and that compiles under this grammar is dispatched here automatically, so its branches and loops actually run, once per chunk, instead diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 3bc2ba17d..f2007ca05 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -26,11 +26,28 @@ are tested independently. Integrating the updated native dependency remains necessary before claiming the precedence change in distributed Python builds. This is **not yet the full v0.1 completion gate**: the wider dtype/semantic audit, -portable-profile validator, artifact schema/loader, and captured-constant export +artifact schema/loader, and captured-constant export are still pending. The raw-source constructor accepts native DSL, not only the draft portable subset. No new dependency or persisted artifact format has been introduced. +Native profile validation is now implemented as `me_validate_portable_dsl()` in +`../miniexpr/src/dsl_portable.c`, with a structured status and optional diagnostic. +It filters native parsed source, checks an explicit homogeneous-input signature, +then compiles with JIT disabled without evaluating a kernel. The corpus runner +uses it before execution. `blosc2.validate_portable_dsl()` exposes the same check +through the extension, with no Python parser/rewriter fallback; older native builds +report `runtime_unsupported`. The local editable environment is rebuilt against +the authoritative checkout via `FETCHCONTENT_SOURCE_DIR_MINIEXPR`; the repository +dependency pin remains unchanged. The draft spec now documents the exact initial +feature filter and distinguishes validation from runtime constraints, backend +compatibility, numerical-accuracy certification, and trust/resource policies. + +Validation for this slice passed: 137 full native tests, 70 conformance/validation +tests with bundled TCC disabled, 24 AddressSanitizer checks, and 210 focused Python +tests with standalone native-runner comparisons enabled. Ruff and diff whitespace +checks passed; the native builds emitted no new compiler warnings. + The next conformance slice expands the shared corpus with explicit input/output dtypes and expected compile/evaluation outcomes. It covers all five candidate dtypes, exact int64 boundaries and comparisons above `2**53`, signed diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 39008fd0b..8869e71af 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -586,7 +586,14 @@ def _raise(exc): from .c2array import c2context, C2Array, C2NDSource, ChunkAlreadyWritten, URLPath -from .dsl_kernel import DSLSyntaxError, DSLKernel, dsl_kernel, validate_dsl, validate_dsl_jit +from .dsl_kernel import ( + DSLSyntaxError, + DSLKernel, + dsl_kernel, + validate_dsl, + validate_dsl_jit, + validate_portable_dsl, +) from .jit_config import get_jit_options, jit_options, set_jit_options from .lazyexpr import ( LazyExpr, @@ -908,6 +915,7 @@ def _raise(exc): "Index", "LazyArray", "DSLKernel", + "validate_portable_dsl", "DSLSyntaxError", "LazyExpr", "LazyUDF", diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index 64affa623..e8ad59f4d 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -961,6 +961,80 @@ cdef inline object _numpy_dtype_from_me_dtype(me_dtype dt): return None +cdef extern from *: + """ + #include "miniexpr.h" + #include + static int b2_validate_portable_dsl(const char *source, const char *version, + const me_variable *inputs, int ninputs, me_dtype output_dtype, + int *line, int *column, char *message, size_t capacity) { + #ifdef ME_PORTABLE_DSL_VERSION + me_portable_error error = {0}; + int rc = me_validate_portable_dsl(source, version, inputs, ninputs, + output_dtype, &error); + *line = error.line; + *column = error.column; + snprintf(message, capacity, "%s", error.message); + return rc; + #else + *line = *column = 0; + snprintf(message, capacity, "miniexpr was built without portable DSL validation support"); + return -100; + #endif + } + """ + int b2_validate_portable_dsl(const char *source, const char *version, + const me_variable *inputs, int ninputs, me_dtype output_dtype, + int *line, int *column, char *message, size_t capacity) + + +def validate_portable_dsl_source(source, input_dtypes, output_dtype, version): + """Native-only draft profile validation; no arrays, JIT, or Python execution.""" + cdef Py_ssize_t n = len(input_dtypes) + if n > 128: + raise ValueError("Too many portable DSL inputs (maximum 128)") + cdef me_variable *variables = NULL + cdef list names = [name.encode("utf-8") for name in input_dtypes] + cdef bytes source_bytes = source.encode("utf-8") + cdef bytes version_bytes = version.encode("utf-8") + cdef bytes name_bytes + cdef np.dtype dtype + cdef me_dtype result_dtype + cdef int rc, line = 0, column = 0 + cdef char message[256] + cdef Py_ssize_t i + if n: + variables = calloc(n, sizeof(me_variable)) + if variables == NULL: + raise MemoryError() + try: + for i, value in enumerate(input_dtypes.values()): + name_bytes = names[i] + variables[i].name = name_bytes + dtype = np.dtype(value) + try: + variables[i].dtype = _me_dtype_from_numpy_dtype(dtype) + except TypeError: + variables[i].dtype = ME_AUTO + dtype = np.dtype(output_dtype) + try: + result_dtype = _me_dtype_from_numpy_dtype(dtype) + except TypeError: + result_dtype = ME_AUTO + rc = b2_validate_portable_dsl(source_bytes, version_bytes, variables, n, + result_dtype, &line, &column, message, sizeof(message)) + statuses = { + 0: "success", -1: "unsupported_version", -2: "invalid_signature", + -3: "invalid_source", -4: "unsupported_feature", -5: "out_of_memory", + -100: "runtime_unsupported", + } + return {"valid": rc == 0, "status": statuses.get(rc, "native_error"), + "line": line, "column": column, + "error": None if rc == 0 else (message).decode("utf-8", "replace")} + finally: + free(variables) + + def me_output_dtype(expression, operands): """Ask miniexpr what dtype *expression* would produce over *operands*. diff --git a/src/blosc2/dsl_kernel.py b/src/blosc2/dsl_kernel.py index d679c5d43..9abf86de9 100644 --- a/src/blosc2/dsl_kernel.py +++ b/src/blosc2/dsl_kernel.py @@ -1021,6 +1021,43 @@ def validate_dsl(func): } +def validate_portable_dsl(source, input_dtypes, output_dtype, *, language_version="0.1"): + """Check raw native source and a typed signature against the draft portable profile. + + Return a dictionary with ``valid``, ``status``, ``line``, ``column``, and + ``error``. Validation neither executes a kernel nor invokes a JIT compiler. + It does not normalize frontend syntax, inspect globals, or certify runtime + input ranges, target compatibility, or sandbox safety. Native support is + required; older builds report ``runtime_unsupported`` without Python fallback. + + ``input_dtypes`` maps parameter names to explicit dtype descriptions. Parameter + binding is by name; mapping order may differ from source order. Dtypes describe + logical values, not the storage layout of any actual input arrays. + """ + from collections.abc import Mapping + + from . import blosc2_ext + + if not isinstance(source, str) or not isinstance(language_version, str): + raise TypeError("source and language_version must be strings") + if not isinstance(input_dtypes, Mapping): + raise TypeError("input_dtypes must map parameter names to explicit dtypes") + if "\x00" in source or "\x00" in language_version: + raise ValueError("Portable DSL source and version must not contain NUL characters") + if any(not isinstance(name, str) or "\x00" in name for name in input_dtypes): + raise ValueError("Portable DSL input names must be strings without NUL characters") + validator = getattr(blosc2_ext, "validate_portable_dsl_source", None) + if validator is None: + return { + "valid": False, + "status": "runtime_unsupported", + "line": 0, + "column": 0, + "error": "Rebuild miniexpr and the extension with portable DSL validation support", + } + return validator(source, input_dtypes, output_dtype, language_version) + + def validate_dsl_jit(func, operands, out_dtype, *, shape=(64,), chunks=None, blocks=None): """Report whether a DSL kernel JIT-compiles (vs. interpreter fallback). diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 519172a53..217c426fa 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -148,3 +148,98 @@ def test_from_source_invalid_native_body(): kernel = DSLKernel.from_source("def kernel(x):\n return x[0]\n") with pytest.raises((NotImplementedError, RuntimeError)): blosc2.lazyudf(kernel, (np.arange(4, dtype=np.float64),), dtype=np.float64)[:] + + +@pytest.fixture +def portable_validator(): + info = blosc2.validate_portable_dsl("def k(x):\n return x\n", {"x": "float64"}, "float64") + if info["status"] == "runtime_unsupported": + pytest.skip("Rebuild with miniexpr portable validation support") + assert info == {"valid": True, "status": "success", "line": 0, "column": 0, "error": None} + return blosc2.validate_portable_dsl + + +@pytest.mark.parametrize("case", CASES) +def test_portable_profile_corpus(portable_validator, case): + if not CORPUS.is_dir(): + pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") + words = (CORPUS / f"{case}.txt").read_text().split() + outcome, input_type, output_type = words[:3] + nvars = int(words[4]) + inputs = dict.fromkeys(words[5 : 5 + nvars], input_type) + info = portable_validator((CORPUS / f"{case}.dsl").read_text(), inputs, output_type) + assert info["valid"] == (outcome != "compile_error"), info + + +@pytest.mark.parametrize( + ("body", "status"), + [ + ("return sum(x)", "unsupported_feature"), + ("return np.sin(x)", "unsupported_feature"), + ("return x.lower()", "unsupported_feature"), + ("return x[0]", "unsupported_feature"), + ("return _flat_idx + x", "unsupported_feature"), + ("return callback(x)", "unsupported_feature"), + ("return CAPTURE + x", "invalid_source"), + ("return sin()", "invalid_source"), + ], +) +def test_portable_profile_rejections(portable_validator, body, status): + info = portable_validator(f"def k(x):\n {body}\n", {"x": "float64"}, "float64") + assert not info["valid"] + assert info["status"] == status + assert info["error"] + assert info["line"] == 2 + assert info["column"] > 0 + + +def test_portable_profile_signature_and_version(portable_validator): + source = "def k(x, y):\n return x + y\n" + inputs = {"y": "float64", "x": "float64"} + assert portable_validator(source, inputs, "float64")["valid"] + assert portable_validator(source, inputs, "float64", language_version="99")["status"] == ( + "unsupported_version" + ) + assert portable_validator(source, {"x": "float64"}, "float64")["status"] == "invalid_signature" + assert portable_validator(source, {"x": "float64", "y": "int32"}, "float64")["status"] == ( + "unsupported_feature" + ) + assert portable_validator(source, inputs, "complex128")["status"] == "unsupported_feature" + + +def test_portable_profile_never_executes_or_compiles_jit(portable_validator, monkeypatch, tmp_path): + cache = tmp_path / "jit-cache" + target = tmp_path / "executed" + monkeypatch.setenv("CC", "/no/compiler") + monkeypatch.setenv("ME_DSL_JIT_COMPILER", "cc") + monkeypatch.setenv("ME_DSL_JIT", "1") + monkeypatch.setenv("ME_DSL_JIT_CACHE_DIR", str(cache)) + # This would reach the iteration cap if executed. Validation must still succeed. + source = "# me:compiler=cc\ndef k(x):\n while 1:\n pass\n return x\n" + assert portable_validator(source, {"x": "float64"}, "float64")["valid"] + source = f"def k(x):\n return open({str(target)!r}, 'w')\n" + assert not portable_validator(source, {"x": "float64"}, "float64")["valid"] + assert not target.exists() + assert not cache.exists() + + +def test_portable_profile_missing_native_support(monkeypatch): + from blosc2 import blosc2_ext + + monkeypatch.delattr(blosc2_ext, "validate_portable_dsl_source", raising=False) + info = blosc2.validate_portable_dsl("def k(x):\n return x\n", {"x": "float64"}, "float64") + assert not info["valid"] + assert info["status"] == "runtime_unsupported" + + +@pytest.mark.parametrize( + ("source", "inputs", "version"), + [ + ("def k(x):\n return x\n\x00", {"x": "float64"}, "0.1"), + ("def k(x):\n return x\n", {"x\x00": "float64"}, "0.1"), + ("def k(x):\n return x\n", {"x": "float64"}, "0.1\x00"), + ], +) +def test_portable_profile_rejects_nul(source, inputs, version): + with pytest.raises(ValueError, match="NUL"): + blosc2.validate_portable_dsl(source, inputs, "float64", language_version=version) From f2380d6c7bf21a1c581bb3ad6e62e8f8c8288d55 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 17:01:08 +0200 Subject: [PATCH 04/30] Record native portable artifact milestone --- plans/portable-DSLs-first-version.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index f2007ca05..5e3ea65c9 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -26,7 +26,7 @@ are tested independently. Integrating the updated native dependency remains necessary before claiming the precedence change in distributed Python builds. This is **not yet the full v0.1 completion gate**: the wider dtype/semantic audit, -artifact schema/loader, and captured-constant export +artifact packaging/integration, Python artifact import, and captured-constant export are still pending. The raw-source constructor accepts native DSL, not only the draft portable subset. No new dependency or persisted artifact format has been introduced. @@ -48,6 +48,29 @@ tests with bundled TCC disabled, 24 AddressSanitizer checks, and 210 focused Pyt tests with standalone native-runner comparisons enabled. Ruff and diff whitespace checks passed; the native builds emitted no new compiler warnings. +P3 now has a candidate artifact schema in `../miniexpr/doc/dsl-spec/artifact-0.1.md` +and a separate optional `miniexpr_artifact` C adapter. With user approval, it uses +yyjson 0.12.0 pinned to `8b4a38dc994a110abaec8a400615567bd996105f`; the raw compiler +does not fetch/link yyjson when `MINIEXPR_BUILD_ARTIFACT` is OFF (the default). +The adapter validates strict JSON structure and duplicate keys, versions and +capabilities, typed scalar encodings, entry point and exact binding coverage, +and portable-profile compilation. It owns source/names/scalars, binds runtime +buffers by name, broadcasts constants in bounded tiles, and separates load, +binding, native evaluation, and allocation failures. Required strict FP semantics +are made explicit on an internal source copy when no FP pragma is supplied; +retained compiler pragmas remain preferences rather than compatibility requirements. +The hand-authored affine artifact runs in the standalone C runner without Python. +The schema/profile remain experimental; native adapter packaging, Python artifact +export/import (P4), captures, and the remaining semantic audit are still pending. + +P3 validation passed: 143 full native tests, 75 conformance/validation/artifact +checks with bundled TCC disabled, 29 AddressSanitizer checks, and 210 focused +Python regression tests. A fresh static-only raw-compiler build without artifact, +SLEEF, TCC, or Accelerate dependencies passed 70 conformance/validation checks; +yyjson was not fetched. The adapter and tests also passed `-Wall -Wextra -Werror` +syntax checks. A test-only deprecated `sprintf` warning was fixed; final native +builds emitted no new compiler warnings. + The next conformance slice expands the shared corpus with explicit input/output dtypes and expected compile/evaluation outcomes. It covers all five candidate dtypes, exact int64 boundaries and comparisons above `2**53`, signed From 6967517a045612a1b1af6b436169fc9158f96cec Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 17:16:08 +0200 Subject: [PATCH 05/30] Add portable DSL artifact export and native import --- CMakeLists.txt | 5 + doc/reference/dsl_syntax.md | 80 ++++++ examples/ndarray/portable_artifact.py | 25 ++ plans/portable-DSLs-first-version.md | 33 ++- src/blosc2/__init__.py | 3 + src/blosc2/blosc2_ext.pyx | 114 ++++++++ src/blosc2/dsl_artifact_bridge.h | 68 +++++ src/blosc2/dsl_kernel.py | 32 ++- src/blosc2/portable_kernel.py | 357 ++++++++++++++++++++++++++ tests/test_portable_artifact.py | 351 +++++++++++++++++++++++++ 10 files changed, 1063 insertions(+), 5 deletions(-) create mode 100644 examples/ndarray/portable_artifact.py create mode 100644 src/blosc2/dsl_artifact_bridge.h create mode 100644 src/blosc2/portable_kernel.py create mode 100644 tests/test_portable_artifact.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e9c5a078b..fc66a804a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,7 @@ add_custom_command( COMMAND Python::Interpreter -m cython "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx" --output-file blosc2_ext.c DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx" + "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/dsl_artifact_bridge.h" VERBATIM) add_custom_command( @@ -147,6 +148,10 @@ FetchContent_MakeAvailable(miniexpr) # Link against miniexpr static library target_link_libraries(blosc2_ext PRIVATE miniexpr_static) +if(TARGET miniexpr_artifact) + target_link_libraries(blosc2_ext PRIVATE miniexpr_artifact) + target_compile_definitions(blosc2_ext PRIVATE B2_HAVE_PORTABLE_ARTIFACT=1) +endif() if(APPLE) target_link_libraries(blosc2_ext PRIVATE "-framework Accelerate") endif() diff --git a/doc/reference/dsl_syntax.md b/doc/reference/dsl_syntax.md index 6dd9dc375..ad88a6987 100644 --- a/doc/reference/dsl_syntax.md +++ b/doc/reference/dsl_syntax.md @@ -66,6 +66,86 @@ than attempting Python validation. Native dependency integration for this experimental API is pending; development builds can use the local miniexpr source via CMake's `FETCHCONTENT_SOURCE_DIR_MINIEXPR` override. +### Portable artifact export and import (experimental) + +Export with explicit input/output dtypes. Scalar globals and closure values are +snapshotted into typed manifest constants, not inserted as Python literals: + +```python +SCALE = 2.0 + + +@blosc2.dsl_kernel +def scaled(x): + return x * SCALE + + +artifact = scaled.export({"x": "float64"}, "float64") +SCALE = 99.0 # Does not change the saved artifact. +kernel = blosc2.PortableKernel.from_json(artifact, jit=False) +values = kernel.evaluate({"x": np.arange(4, dtype=np.float64)}) +# values: array([0., 2., 4., 6.]) +``` + +`export()` returns deterministic UTF-8 JSON text; save it with an ordinary file +write. `from_json()` accepts text or bytes, not a pathname or decoded mapping. +Import uses the same native JSON loader as C and does not need the author's module, +function, decorators, globals, Python source reconstruction, or frontend rewrites. +`evaluate()` returns a new NumPy array; `blosc2.asarray(values)` can materialize +compressed storage. This first adapter is eager, not a `lazyudf` integration. + +Python `bool`, `int`, and `float` captures infer `bool`, `int64`, and `float64`. +Supported NumPy scalar types retain their dtype. Use +`capture_dtypes={"SCALE": "float32"}` to explicitly specialize a capture; finite +float narrowing rounds to the chosen dtype, while overflow and integer precision +loss are rejected. Arrays (including zero-dimensional arrays), objects, unsupported +scalar types, unresolved names, and external callbacks cannot be captures. +Export does not call the kernel or capture conversion hooks. It reuses supported +authoring normalization, such as `np.sin(x)`, before native profile validation. +Strings, arbitrary globals, and unsupported functions do not become portable. + +Native-only kernels can bind existing parameters as constants explicitly: + +```python +author = blosc2.DSLKernel.from_source( + "# me:compiler=cc\ndef scaled(x, scale):\n return x * scale\n" +) +artifact = author.export({"x": "float64"}, "float64", constants={"scale": 2.0}) +``` + +Every source parameter requires exactly one runtime input or constant binding. +The draft profile requires homogeneous parameter dtypes, including captures; +output may have a different supported dtype. Manifest float constants preserve +IEEE bit patterns, including signed zero and non-finite values; integer constants +are range-checked decimal strings. Comments and retained native pragmas survive +parameterization. Required strict FP semantics are independent of host defaults; +compiler pragmas remain preferences, and `jit=False` disables JIT independently. + +Runtime arrays are bound by name in any mapping order and must have the exact +declared logical dtype and common shape. The adapter converts endian order, +alignment, and strides, but does not broadcast arrays or silently change their +dtype. Constants broadcast in bounded native workspace. Constant-only kernels +require `evaluate({}, shape=(...))`; empty arrays validate bindings but do not +execute. `PortableArtifactError` provides `status`, `native_status`, `line`, and +`column` for malformed artifacts, unsupported requirements, source, binding, and +evaluation failures. The artifact format is not a sandbox or an arbitrary-input +numerical-correctness guarantee. + +Native dependency packaging is still pending. For development builds, use both +`FETCHCONTENT_SOURCE_DIR_MINIEXPR` and `MINIEXPR_BUILD_ARTIFACT=ON`, for example: + +```sh +conda run -n blosc2 python -m pip install -e . --no-build-isolation --no-deps \ + --config-settings=cmake.define.FETCHCONTENT_SOURCE_DIR_MINIEXPR=/path/to/miniexpr \ + --config-settings=cmake.define.MINIEXPR_BUILD_ARTIFACT=ON +``` + +The optional adapter links pinned yyjson only when enabled. Builds without the +adapter remain usable and report `NotImplementedError` for artifact operations, +without attempting Python validation/execution as a fallback. The native candidate +format and shared fixtures live in miniexpr's `doc/dsl-spec/artifact-0.1.md` and +`tests/portable-artifacts/`. + `@blosc2.jit` auto-detects this DSL: a decorated function whose body contains an `if`/`for`/`while` and that compiles under this grammar is dispatched here automatically, so its branches and loops actually run, once per chunk, instead diff --git a/examples/ndarray/portable_artifact.py b/examples/ndarray/portable_artifact.py new file mode 100644 index 000000000..363bcdd9f --- /dev/null +++ b/examples/ndarray/portable_artifact.py @@ -0,0 +1,25 @@ +####################################################################### +# Copyright (c) 2026, Blosc Development Team +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### +"""Snapshot a scalar capture and execute the portable artifact without its author.""" + +import numpy as np + +import blosc2 + +SCALE = 2.0 + + +@blosc2.dsl_kernel +def scaled(x): + return x * SCALE + + +if __name__ == "__main__": + artifact = scaled.export({"x": "float64"}, "float64") + SCALE = 99.0 + imported = blosc2.PortableKernel.from_json(artifact, jit=False) + values = imported.evaluate({"x": np.arange(4, dtype=np.float64)}) + np.testing.assert_array_equal(values, [0, 2, 4, 6]) + print(values) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 5e3ea65c9..01bbef49e 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -26,7 +26,7 @@ are tested independently. Integrating the updated native dependency remains necessary before claiming the precedence change in distributed Python builds. This is **not yet the full v0.1 completion gate**: the wider dtype/semantic audit, -artifact packaging/integration, Python artifact import, and captured-constant export +artifact packaging/integration and the remaining semantic audit are still pending. The raw-source constructor accepts native DSL, not only the draft portable subset. No new dependency or persisted artifact format has been introduced. @@ -60,8 +60,8 @@ binding, native evaluation, and allocation failures. Required strict FP semantic are made explicit on an internal source copy when no FP pragma is supplied; retained compiler pragmas remain preferences rather than compatibility requirements. The hand-authored affine artifact runs in the standalone C runner without Python. -The schema/profile remain experimental; native adapter packaging, Python artifact -export/import (P4), captures, and the remaining semantic audit are still pending. +The schema/profile remain experimental; native adapter packaging and the remaining +semantic audit are still pending. P4 integration is described below. P3 validation passed: 143 full native tests, 75 conformance/validation/artifact checks with bundled TCC disabled, 29 AddressSanitizer checks, and 210 focused @@ -71,6 +71,33 @@ yyjson was not fetched. The adapter and tests also passed `-Wall -Wextra -Werror syntax checks. A test-only deprecated `sprintf` warning was fixed; final native builds emitted no new compiler warnings. +P4 now exposes `DSLKernel.export()` and `PortableKernel.from_json()` / +`evaluate()` in Python. Export uses the unspecialized author's DSL source and +snapshots exact supported scalar globals/closure values into collision-free +parameters and typed constants. Explicit parameter constants and capture dtype +overrides are supported; no user conversion hooks or kernel calls are used. +Input/output dtypes are explicit and portable native validation plus the C loader +validate every exported artifact. Import calls that same loader through an owned +Cython handle, without Python source/JSON parsing or frontend preparation. +Runtime input names/dtypes/shapes are checked, host storage is adapted, and native +constant broadcast/evaluation is reused. NumPy-alias rewriting now excludes local +and closure shadowing so export cannot silently reinterpret a local attribute call. +The extension links the adapter only if the optional native target is available; +disabled/older builds report unsupported artifact operations. Local builds use +the source override and `MINIEXPR_BUILD_ARTIFACT=ON`; the repository dependency pin +is unchanged because native dependency publication/integration is still pending. +Artifact execution is currently eager NumPy output, not lazyudf/container integration. + +P4 validation passed: 240 focused Python tests with the adapter enabled, including +Python export -> standalone C execution, fresh-process import without the author, +typed scalar boundaries/NaN payloads, snapshot/collision/closure handling, storage +adaptation, native diagnostics, and concurrent interpreter/TCC evaluations. +An actual adapter-disabled editable build passed 212 tests with 28 artifact-only +cases skipped and verified `NotImplementedError` without a Python fallback. +The environment was restored to the adapter-enabled local build. The native +conformance/profile/artifact selection passed 99 checks; Ruff, the standalone +example, and whitespace checks also passed. + The next conformance slice expands the shared corpus with explicit input/output dtypes and expected compile/evaluation outcomes. It covers all five candidate dtypes, exact int64 boundaries and comparisons above `2**53`, signed diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 8869e71af..3fa503c49 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -594,6 +594,7 @@ def _raise(exc): validate_dsl_jit, validate_portable_dsl, ) +from .portable_kernel import PortableArtifactError, PortableKernel from .jit_config import get_jit_options, jit_options, set_jit_options from .lazyexpr import ( LazyExpr, @@ -916,6 +917,8 @@ def _raise(exc): "LazyArray", "DSLKernel", "validate_portable_dsl", + "PortableKernel", + "PortableArtifactError", "DSLSyntaxError", "LazyExpr", "LazyUDF", diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index e8ad59f4d..7b74a63aa 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -1035,6 +1035,120 @@ def validate_portable_dsl_source(source, input_dtypes, output_dtype, version): free(variables) +cdef extern from "dsl_artifact_bridge.h": + ctypedef struct b2_artifact_input: + const char *name + me_dtype dtype + const void *data + size_t nitems + ctypedef struct b2_artifact_error: + int native_status + int line + int column + char message[256] + int b2_artifact_available() + int b2_artifact_load(const char *, size_t, int, void **, b2_artifact_error *) nogil + int b2_artifact_eval(const void *, const b2_artifact_input *, int, void *, size_t, + b2_artifact_error *) nogil + void b2_artifact_free(void *) noexcept nogil + const char *b2_artifact_source(const void *) + const char *b2_artifact_entry(const void *) + int b2_artifact_ninputs(const void *) + const char *b2_artifact_name(const void *, int) + me_dtype b2_artifact_dtype(const void *, int) + me_dtype b2_artifact_output(const void *) + int b2_artifact_jit(const void *) + + +def portable_artifact_available(): + return bool(b2_artifact_available()) + + +cdef void raise_artifact_error(int rc, b2_artifact_error *error) except *: + from blosc2.portable_kernel import PortableArtifactError + + if rc == -100: + raise NotImplementedError("Build with miniexpr portable artifact support") + if rc == -6: + raise MemoryError((error.message).decode("utf-8", "replace")) + statuses = {-1: "invalid_artifact", -2: "unsupported_requirement", -3: "invalid_source", + -4: "binding_error", -5: "evaluation_error"} + raise PortableArtifactError( + (error.message).decode("utf-8", "replace"), + status=statuses.get(rc, "native_error"), native_status=error.native_status, + line=error.line, column=error.column) + + +cdef class PortableArtifactHandle: + """Owned immutable native handle; no Python artifact decoding or source execution.""" + cdef void *_handle + + def __cinit__(self, bytes artifact, int jit_mode): + cdef b2_artifact_error error + cdef int rc + cdef const char *json = artifact + cdef size_t size = len(artifact) + self._handle = NULL + with nogil: + rc = b2_artifact_load(json, size, jit_mode, &self._handle, &error) + if rc: + raise_artifact_error(rc, &error) + + def __dealloc__(self): + b2_artifact_free(self._handle) + + def info(self): + cdef int i + return { + "source": (b2_artifact_source(self._handle)).decode("utf-8"), + "entry_point": (b2_artifact_entry(self._handle)).decode("utf-8"), + "inputs": {(b2_artifact_name(self._handle, i)).decode("utf-8"): + _numpy_dtype_from_me_dtype(b2_artifact_dtype(self._handle, i)) + for i in range(b2_artifact_ninputs(self._handle))}, + "output_dtype": _numpy_dtype_from_me_dtype(b2_artifact_output(self._handle)), + "jit": bool(b2_artifact_jit(self._handle)), + } + + def evaluate(self, inputs, shape): + cdef Py_ssize_t n = len(inputs) + if n > 128: + raise ValueError("Too many portable artifact inputs") + cdef b2_artifact_input *bindings = NULL + cdef b2_artifact_error error + cdef np.ndarray array, output + cdef list names = [key.encode("utf-8") for key in inputs] + cdef list arrays = list(inputs.values()) + cdef bytes name + cdef int rc + cdef Py_ssize_t i + cdef size_t count + output = np.empty(shape, dtype=_numpy_dtype_from_me_dtype(b2_artifact_output(self._handle))) + count = output.size + if n: + bindings = calloc(n, sizeof(b2_artifact_input)) + if bindings == NULL: + raise MemoryError() + try: + for i in range(n): + array = arrays[i] + if not array.flags.c_contiguous or not array.flags.aligned or not array.dtype.isnative: + raise ValueError("Native artifact inputs must be aligned, contiguous, and host-endian") + name = names[i] + if b'\x00' in name: + raise ValueError("Input names must not contain NUL") + bindings[i].name = name + bindings[i].dtype = _me_dtype_from_numpy_dtype(array.dtype) + bindings[i].data = np.PyArray_DATA(array) + bindings[i].nitems = array.size + with nogil: + rc = b2_artifact_eval(self._handle, bindings, n, np.PyArray_DATA(output), count, &error) + if rc: + raise_artifact_error(rc, &error) + return output + finally: + free(bindings) + + def me_output_dtype(expression, operands): """Ask miniexpr what dtype *expression* would produce over *operands*. diff --git a/src/blosc2/dsl_artifact_bridge.h b/src/blosc2/dsl_artifact_bridge.h new file mode 100644 index 000000000..cfc28fc2a --- /dev/null +++ b/src/blosc2/dsl_artifact_bridge.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2026, Blosc Development Team + * SPDX-License-Identifier: BSD-3-Clause + * Optional native artifact support; old miniexpr builds remain usable. */ +#ifndef B2_DSL_ARTIFACT_BRIDGE_H +#define B2_DSL_ARTIFACT_BRIDGE_H +#include "miniexpr.h" +#include +#include + +#ifdef B2_HAVE_PORTABLE_ARTIFACT +#include "miniexpr_artifact.h" +typedef me_artifact_input b2_artifact_input; +typedef me_artifact_error b2_artifact_error; +static int b2_artifact_available(void) { return 1; } +static int b2_artifact_load(const char *json, size_t size, int jit, void **out, b2_artifact_error *error) { + me_artifact *handle = NULL; + int rc = me_artifact_load(json, size, (me_jit_mode)jit, &handle, error); + *out = handle; + return rc; +} +static int b2_artifact_eval(const void *handle, const b2_artifact_input *inputs, int ninputs, + void *output, size_t count, b2_artifact_error *error) { + return me_artifact_eval((const me_artifact *)handle, inputs, ninputs, output, count, error); +} +static void b2_artifact_free(void *handle) { me_artifact_free((me_artifact *)handle); } +static const char *b2_artifact_source(const void *handle) { return me_artifact_source(handle); } +static const char *b2_artifact_entry(const void *handle) { return me_artifact_entry_point(handle); } +static int b2_artifact_ninputs(const void *handle) { return me_artifact_ninputs(handle); } +static const char *b2_artifact_name(const void *handle, int index) { return me_artifact_input_name(handle, index); } +static me_dtype b2_artifact_dtype(const void *handle, int index) { return me_artifact_input_dtype(handle, index); } +static me_dtype b2_artifact_output(const void *handle) { return me_artifact_output_dtype(handle); } +static int b2_artifact_jit(const void *handle) { return me_artifact_has_jit(handle); } +#else +typedef struct { + const char *name; + me_dtype dtype; + const void *data; + size_t nitems; +} b2_artifact_input; +typedef struct { + int native_status; + int line; + int column; + char message[256]; +} b2_artifact_error; +static int b2_artifact_available(void) { return 0; } +static int b2_artifact_load(const char *json, size_t size, int jit, void **out, b2_artifact_error *error) { + (void)json; (void)size; (void)jit; + *out = NULL; + memset(error, 0, sizeof(*error)); + snprintf(error->message, sizeof(error->message), "Build with miniexpr portable artifact support"); + return -100; +} +static int b2_artifact_eval(const void *handle, const b2_artifact_input *inputs, int ninputs, + void *output, size_t count, b2_artifact_error *error) { + (void)handle; (void)inputs; (void)ninputs; (void)output; (void)count; (void)error; + return -100; +} +static void b2_artifact_free(void *handle) { (void)handle; } +static const char *b2_artifact_source(const void *handle) { (void)handle; return NULL; } +static const char *b2_artifact_entry(const void *handle) { (void)handle; return NULL; } +static int b2_artifact_ninputs(const void *handle) { (void)handle; return 0; } +static const char *b2_artifact_name(const void *handle, int index) { (void)handle; (void)index; return NULL; } +static me_dtype b2_artifact_dtype(const void *handle, int index) { (void)handle; (void)index; return ME_AUTO; } +static me_dtype b2_artifact_output(const void *handle) { (void)handle; return ME_AUTO; } +static int b2_artifact_jit(const void *handle) { (void)handle; return 0; } +#endif +#endif diff --git a/src/blosc2/dsl_kernel.py b/src/blosc2/dsl_kernel.py index 9abf86de9..6a4efa76d 100644 --- a/src/blosc2/dsl_kernel.py +++ b/src/blosc2/dsl_kernel.py @@ -890,12 +890,17 @@ def _rewrite_numpy_attr_calls(func, dsl_source, dsl_tree, dsl_func, input_names) any alias, including a bare `numpy` import, is honored). The DSL grammar only accepts bare function-name calls. No-op, returning the inputs unchanged, when there is nothing to rewrite (including when the alias is - shadowed by one of the kernel's own parameter names). + shadowed by a parameter, local, or closure binding). """ + code = getattr(func, "__code__", None) + shadowed = set(input_names) + if code is not None: + shadowed.update(code.co_varnames) + shadowed.update(code.co_freevars) aliases = { name for name, value in getattr(func, "__globals__", {}).items() - if value is numpy and name not in input_names + if value is numpy and name not in shadowed } if not aliases: return dsl_source, dsl_tree, dsl_func @@ -929,6 +934,29 @@ def _input_names_from_signature(func_node: ast.FunctionDef) -> list[str]: raise ValueError("DSL kernel does not support *args/**kwargs/kwonly args") return [a.arg for a in (args.posonlyargs + args.args)] + def export(self, input_dtypes, output_dtype, *, capture_dtypes=None, constants=None, metadata=None): + """Export a typed portable JSON artifact without calling the Python function. + + ``input_dtypes`` maps runtime parameter names to explicit logical dtypes. + Optional ``constants`` supplies scalar values for remaining parameters. + Globals/closure scalars are snapshotted at export and become explicit, + collision-free parameters. ``capture_dtypes`` overrides scalar type + inference (Python bool/int/float infer bool/int64/float64 respectively; + supported NumPy scalars retain their dtype). Arrays, objects, and external + callbacks cannot be captures. The draft requires homogeneous parameter + dtypes. Import via :meth:`blosc2.PortableKernel.from_json`. + """ + from .portable_kernel import export_portable_kernel + + return export_portable_kernel( + self, + input_dtypes, + output_dtype, + capture_dtypes=capture_dtypes, + constants=constants, + metadata=metadata, + ) + def __call__(self, inputs_tuple, output, offset=None): if self.func is None: raise RuntimeError("A native-source DSL kernel cannot execute as a Python function") diff --git a/src/blosc2/portable_kernel.py b/src/blosc2/portable_kernel.py new file mode 100644 index 000000000..f50506b20 --- /dev/null +++ b/src/blosc2/portable_kernel.py @@ -0,0 +1,357 @@ +####################################################################### +# Copyright (c) 2026, Blosc Development Team +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### +"""Experimental portable DSL artifact authoring and native-only execution.""" + +from __future__ import annotations + +import ast +import builtins +import json +import math +import tokenize +from collections.abc import Mapping +from io import StringIO +from types import FunctionType, MappingProxyType + +import numpy as np + +_DTYPES = {"bool", "int32", "int64", "float32", "float64"} +_SCALAR_TYPES = (bool, int, float, np.bool_, np.int32, np.int64, np.float32, np.float64) +_CALLS = {"sin", "cos", "int", "float", "bool", "range"} + + +class PortableArtifactError(ValueError): + """Artifact failure with a stable category and optional native/source diagnostics.""" + + def __init__(self, message, *, status, native_status=0, line=0, column=0): + self.status = status + self.native_status = native_status + self.line = line + self.column = column + location = f" at line {line}, column {column}" if line else "" + super().__init__(f"{status}{location}: {message}") + + +def _logical_dtype(value): + dtype = np.dtype(value) + if dtype.name not in _DTYPES or dtype.subdtype or dtype.fields: + raise PortableArtifactError(f"Unsupported portable dtype {dtype}", status="unsupported_requirement") + return dtype.newbyteorder("=") + + +def _mapping(value, label): + if not isinstance(value, Mapping): + raise TypeError(f"{label} must be a mapping") + if any(not isinstance(name, str) or not name or "\x00" in name for name in value): + raise ValueError(f"{label} names must be nonempty strings without NUL characters") + return dict(value) + + +def _encode_scalar(name, value, dtype=None): + # Exact types only: no ndarray.item(), user __float__/__int__, or other hooks. + if type(value) not in _SCALAR_TYPES: + raise PortableArtifactError( + f"Capture {name!r} must be a supported Python/NumPy scalar, not {type(value).__name__}", + status="unsupported_requirement", + ) + if dtype is None: + if type(value) is bool: + dtype = "bool" + elif type(value) is int: + dtype = "int64" + elif type(value) is float: + dtype = "float64" + else: + dtype = value.dtype + dtype = _logical_dtype(dtype) + if dtype.kind in "bi": + if type(value) in (float, np.float32, np.float64): + raise PortableArtifactError( + f"Capture {name!r} cannot implicitly convert a float to {dtype}", status="binding_error" + ) + integer = int(value) + if dtype.kind == "b": + if integer not in (0, 1): + raise PortableArtifactError(f"Capture {name!r} is not Boolean", status="binding_error") + encoding, encoded = "boolean", bool(integer) + else: + limits = np.iinfo(dtype) + if not limits.min <= integer <= limits.max: + raise PortableArtifactError( + f"Capture {name!r} is out of range for {dtype}", status="binding_error" + ) + encoding, encoded = "decimal", str(integer) + else: + maximum = float(np.finfo(dtype).max) + if type(value) in (bool, int, np.bool_, np.int32, np.int64): + integer = int(value) + # Reject precision loss, rather than standardizing Python literal rounding. + if abs(integer) > maximum or int(np.array(integer, dtype=dtype)) != integer: + raise PortableArtifactError( + f"Capture {name!r} is not exactly representable as {dtype}", status="binding_error" + ) + elif math.isfinite(value) and abs(float(value)) > maximum: + raise PortableArtifactError(f"Capture {name!r} overflows {dtype}", status="binding_error") + scalar = np.array(value, dtype=dtype) + encoding = "ieee754-hex" + encoded = scalar.astype(dtype.newbyteorder(">"), copy=False).tobytes().hex() + return {"name": name, "dtype": dtype.name, "encoding": encoding, "value": encoded} + + +def _parameterize_source(source, replacements, parameters): + """Token edits retain comments/pragmas and never edit strings or substrings.""" + tokens = list(tokenize.generate_tokens(StringIO(source).readline)) + starts = [0] + for line in source.splitlines(keepends=True): + starts.append(starts[-1] + len(line)) + + def offset(position): + return starts[position[0] - 1] + position[1] + + edits = [] + depth = 0 + signature = False + for token in tokens: + if token.type == tokenize.NAME and token.string == "def": + signature = True + if signature and token.string == "(": + depth += 1 + elif signature and token.string == ")": + depth -= 1 + if depth == 0: + if parameters: + edits.append((offset(token.start), offset(token.start), ", ".join(parameters))) + signature = False + elif token.type == tokenize.NAME and token.string in replacements and not signature: + edits.append((offset(token.start), offset(token.end), replacements[token.string])) + for start, end, text in sorted(edits, reverse=True): + source = source[:start] + text + source[end:] + return source + + +def _capture_scope(func): + if not isinstance(func, FunctionType): + raise PortableArtifactError( + "Only ordinary Python functions can export captures", status="unsupported_requirement" + ) + scope = func.__globals__.copy() + for name, cell in zip(func.__code__.co_freevars, func.__closure__ or (), strict=True): + try: + scope[name] = cell.cell_contents + except ValueError as error: + raise PortableArtifactError(f"Unbound capture {name!r}", status="invalid_source") from error + return scope + + +def _check_capture_calls(calls, local, scope): + for call in sorted(calls): + if call not in _CALLS or call in local: + raise PortableArtifactError(f"Unsupported call {call!r}", status="unsupported_requirement") + allowed = (getattr(builtins, call, None), getattr(np, call, None), getattr(math, call, None)) + if call in scope and not any(value is not None and scope[call] is value for value in allowed): + raise PortableArtifactError( + f"External callback {call!r} cannot be exported", status="unsupported_requirement" + ) + + +def _export_captures(func, source, names, capture_dtypes): + scope = _capture_scope(func) + tree = ast.parse(source) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef)) + nodes = list(ast.walk(function)) + local = set(names) | { + node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) + } + calls = { + node.func.id for node in nodes if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + } + _check_capture_calls(calls, local, scope) + loads = {node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)} + occupied = local | loads | {function.name} + replacements = {} + encoded = [] + for name in sorted(loads - local - calls): + if name not in scope: + node = next(node for node in nodes if isinstance(node, ast.Name) and node.id == name) + raise PortableArtifactError( + f"Unresolved capture {name!r}", + status="invalid_source", + line=node.lineno, + column=node.col_offset + 1, + ) + index = len(replacements) + candidate = f"_capture_{index}" + while candidate in occupied: + index += 1 + candidate = f"_capture_{index}" + occupied.add(candidate) + scalar = _encode_scalar(name, scope[name], capture_dtypes.get(name)) + scalar["name"] = candidate + encoded.append(scalar) + replacements[name] = candidate + added = list(replacements.values()) + if names and added: + added[0] = ", " + added[0] + return ( + _parameterize_source(source, replacements, added), + encoded, + set(replacements) & set(capture_dtypes), + ) + + +def export_portable_kernel( + kernel, input_dtypes, output_dtype, *, capture_dtypes=None, constants=None, metadata=None +): + """Implementation of :meth:`blosc2.DSLKernel.export`; does not call the kernel.""" + from .dsl_kernel import DSLKernel, validate_portable_dsl + + inputs = _mapping(input_dtypes, "input_dtypes") + constants = _mapping({} if constants is None else constants, "constants") + capture_dtypes = _mapping({} if capture_dtypes is None else capture_dtypes, "capture_dtypes") + source = kernel.dsl_source + if not isinstance(source, str): + raise PortableArtifactError("Kernel has no exportable DSL source", status="invalid_source") + # This is only a header check; native validation remains authoritative for the body. + header = DSLKernel.from_source(source) + names = header.input_names + if set(inputs) & set(constants) or set(inputs) | set(constants) != set(names): + raise PortableArtifactError( + "Every parameter needs exactly one input or constant binding", status="binding_error" + ) + input_types = {name: _logical_dtype(inputs[name]).name for name in names if name in inputs} + result_type = _logical_dtype(output_dtype).name + encoded = [_encode_scalar(name, value, capture_dtypes.get(name)) for name, value in constants.items()] + used_capture_types = set(constants) & set(capture_dtypes) + + if kernel.func is not None: + source, captures, used_types = _export_captures(kernel.func, source, names, capture_dtypes) + encoded.extend(captures) + used_capture_types.update(used_types) + unused = set(capture_dtypes) - used_capture_types + if unused: + raise PortableArtifactError(f"Unused capture dtype names: {sorted(unused)}", status="binding_error") + signature = input_types | {item["name"]: item["dtype"] for item in encoded} + info = validate_portable_dsl(source, signature, result_type) + if not info["valid"]: + if info["status"] == "runtime_unsupported": + raise NotImplementedError(info["error"]) + raise PortableArtifactError( + info["error"], status=info["status"], line=info["line"], column=info["column"] + ) + if metadata is not None and not isinstance(metadata, Mapping): + raise TypeError("metadata must be a mapping") + manifest = { + "schema_version": "0.1", + "language": {"name": "miniexpr", "version": "0.1"}, + "requires": ["core-scalar"], + "source": source, + "entry_point": header.__name__, + "inputs": [{"name": name, "dtype": dtype} for name, dtype in input_types.items()], + "constants": sorted(encoded, key=lambda item: item["name"]), + "output": {"dtype": result_type, "contract": "scalar-per-element"}, + "semantics": {"fp": "strict"}, + } + if metadata is not None: + manifest["metadata"] = dict(metadata) + artifact = ( + json.dumps(manifest, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + + "\n" + ) + # Use the same loader as import/C, including bounds and metadata validation. + PortableKernel.from_json(artifact, jit=False) + return artifact + + +class PortableKernel: + """An owned, typed native artifact; no originating function or Python fallback. + + Use ``from_json`` to import and ``evaluate`` to return NumPy values. Inputs + are same-shaped logical arrays, bound by name. The experimental profile and + native adapter must be available in the installed extension. + """ + + @classmethod + def from_json(cls, artifact, *, jit=None): + from . import blosc2_ext + + if not isinstance(artifact, str | bytes): + raise TypeError("artifact must be JSON text or UTF-8 bytes, not a path or decoded mapping") + if jit is not None and type(jit) is not bool: + raise TypeError("jit must be True, False, or None") + handle_type = getattr(blosc2_ext, "PortableArtifactHandle", None) + if handle_type is None: + raise NotImplementedError("Build with miniexpr portable artifact support") + try: + data = artifact.encode("utf-8") if isinstance(artifact, str) else artifact + except UnicodeEncodeError as error: + raise PortableArtifactError( + "Artifact text must be valid UTF-8", status="invalid_artifact" + ) from error + handle = handle_type(data, 0 if jit is None else 1 if jit else 2) + instance = cls.__new__(cls) + instance._handle = handle + instance._artifact = data + instance._info = handle.info() + return instance + + @property + def source(self): + return self._info["source"] + + @property + def entry_point(self): + return self._info["entry_point"] + + @property + def input_dtypes(self): + return MappingProxyType(self._info["inputs"]) + + @property + def output_dtype(self): + return self._info["output_dtype"] + + @property + def has_jit(self): + return self._info["jit"] + + def to_json(self): + """Return the original validated JSON text, without dropping metadata.""" + return self._artifact.decode("utf-8") + + def evaluate(self, inputs, *, shape=None): + """Evaluate named arrays into a new NumPy array, preserving their common shape. + + No implicit dtype conversion or array broadcasting is allowed. Host endian, + alignment, and strides are normalized into temporary contiguous buffers. + Constant-only kernels require an explicit output ``shape`` (possibly empty). + """ + inputs = _mapping(inputs, "inputs") + if set(inputs) != set(self.input_dtypes): + raise PortableArtifactError("Missing or extra runtime inputs", status="binding_error") + if shape is not None: + if isinstance(shape, int): + shape = (shape,) + else: + shape = tuple(shape) + if any(type(size) is not int or size < 0 for size in shape): + raise ValueError("shape must contain nonnegative Python integers") + arrays = {} + for name, value in inputs.items(): + array = np.asarray(value) + dtype = self.input_dtypes[name] + if array.dtype.newbyteorder("=") != dtype: + raise PortableArtifactError( + f"Input {name!r} must have dtype {dtype}, not {array.dtype}", status="binding_error" + ) + if shape is None: + shape = array.shape + if array.shape != shape: + raise PortableArtifactError( + "All input shapes must match the output shape", status="binding_error" + ) + arrays[name] = np.require(array, dtype=dtype, requirements=["C", "A"]) + if shape is None: + raise ValueError("Constant-only kernels require an explicit shape") + return self._handle.evaluate(arrays, shape) diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py new file mode 100644 index 000000000..1e35492ef --- /dev/null +++ b/tests/test_portable_artifact.py @@ -0,0 +1,351 @@ +####################################################################### +# Copyright (c) 2026, Blosc Development Team +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### +import ast +import json +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +import blosc2 + +CORPUS = Path( + os.environ.get( + "MINIEXPR_PORTABLE_CORPUS", Path(__file__).resolve().parents[2] / "miniexpr/tests/portable-dsl" + ) +) +ARTIFACTS = CORPUS.parent / "portable-artifacts" +SCALE = 2.0 +BIAS = -1.0 + + +@blosc2.dsl_kernel +def affine(x): + return x * SCALE + BIAS + + +@pytest.fixture +def native_artifacts(): + from blosc2 import blosc2_ext + + available = getattr(blosc2_ext, "portable_artifact_available", lambda: False) + if not available(): + pytest.skip("Rebuild with miniexpr artifact support enabled") + + +def test_export_capture_snapshot(native_artifacts, monkeypatch): + artifact = affine.export({"x": "float64"}, "float64", metadata={"producer": "test"}) + assert artifact == affine.export({"x": "float64"}, "float64", metadata={"producer": "test"}) + decoded = json.loads(artifact) + assert "SCALE" not in decoded["source"] + assert "BIAS" not in decoded["source"] + assert len(decoded["constants"]) == 2 + monkeypatch.setitem(affine.func.__globals__, "SCALE", 99.0) + monkeypatch.setitem(affine.func.__globals__, "BIAS", 10.0) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + x = np.arange(600, dtype=np.float64).reshape(20, 30) + np.testing.assert_array_equal(kernel.evaluate({"x": x}), 2 * x - 1) + assert not kernel.has_jit + assert kernel.entry_point == "affine" + assert kernel.output_dtype == np.dtype("float64") + assert kernel.input_dtypes == {"x": np.dtype("float64")} + assert kernel.to_json() == artifact + with pytest.raises(TypeError): + kernel.input_dtypes["x"] = np.dtype("int64") + + +def test_capture_closure_and_frontend_sugar(native_artifacts): + scale = np.float32(2.0) + + @blosc2.dsl_kernel + def trig(x): + return np.sin(x) * scale + + artifact = trig.export({"x": "float32"}, "float32") + scale = np.float32(99.0) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + x = np.array([0.0, 0.25, 1.0], dtype=np.float32) + np.testing.assert_allclose(kernel.evaluate({"x": x}), np.sin(x) * 2, rtol=2e-6) + assert "np.sin" not in kernel.source + + +def test_capture_collision_and_comments(native_artifacts): + scale = 2.0 + + @blosc2.dsl_kernel + def collision(x): + """scale must not be rewritten in strings.""" + _capture_0 = x # scale stays in this comment + return _capture_0 * scale + + artifact = collision.export({"x": "float64"}, "float64") + decoded = json.loads(artifact) + assert decoded["constants"][0]["name"] == "_capture_1" + assert "# scale stays in this comment" in decoded["source"] + assert '"""scale must not be rewritten in strings."""' in decoded["source"] + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + np.testing.assert_array_equal(kernel.evaluate({"x": np.arange(3, dtype=np.float64)}), [0, 2, 4]) + + +@pytest.mark.parametrize( + ("dtype", "value", "encoding", "encoded"), + [ + ("bool", True, "boolean", True), + ("int32", np.int32(-(2**31)), "decimal", "-2147483648"), + ("int64", 2**63 - 1, "decimal", "9223372036854775807"), + ("float32", np.float32(-0.0), "ieee754-hex", "80000000"), + ("float64", -0.0, "ieee754-hex", "8000000000000000"), + ("float64", np.inf, "ieee754-hex", "7ff0000000000000"), + ("float64", np.nan, "ieee754-hex", "7ff8000000000000"), + ], +) +def test_constant_encodings(native_artifacts, dtype, value, encoding, encoded): + source = "# me:compiler=cc\ndef constant(value):\n return value\n" + author = blosc2.DSLKernel.from_source(source) + artifact = author.export({}, dtype, constants={"value": value}) + scalar = json.loads(artifact)["constants"][0] + assert scalar == {"name": "value", "dtype": dtype, "encoding": encoding, "value": encoded} + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + assert kernel.source == source + result = kernel.evaluate({}, shape=(600,)) + assert result.dtype == np.dtype(dtype) + np.testing.assert_array_equal(result, np.full(600, value, dtype=dtype)) + if encoded in ("80000000", "8000000000000000"): + assert np.signbit(result).all() + assert kernel.evaluate({}, shape=(0,)).shape == (0,) + with pytest.raises(ValueError, match="explicit shape"): + kernel.evaluate({}) + + +def test_explicit_capture_dtype(native_artifacts): + integer = 2 + + @blosc2.dsl_kernel + def k(x): + return x * integer + + with pytest.raises(blosc2.PortableArtifactError, match="mixed input dtypes"): + k.export({"x": "float64"}, "float64") + artifact = k.export({"x": "float64"}, "float64", capture_dtypes={"integer": "float64"}) + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(artifact, jit=False).evaluate({"x": np.array([3.0])}), [6.0] + ) + with pytest.raises(blosc2.PortableArtifactError, match="Unused capture dtype"): + k.export({"x": "float64"}, "float64", capture_dtypes={"unused": "float64"}) + + +@pytest.mark.parametrize("value", [np.array(2.0), np.arange(2), object(), "hello"]) +def test_unsupported_captures(native_artifacts, value): + capture = value + + @blosc2.dsl_kernel + def k(x): + return x + capture + + with pytest.raises(blosc2.PortableArtifactError, match="Capture 'capture'"): + k.export({"x": "float64"}, "float64") + + +def test_never_calls_capture_hooks_or_functions(native_artifacts): + class Trap: + def __float__(self): + pytest.fail("capture __float__ executed") + + def item(self): + pytest.fail("capture item executed") + + capture = Trap() + + @blosc2.dsl_kernel + def k(x): + return x + capture + + with pytest.raises(blosc2.PortableArtifactError, match="Capture 'capture'"): + k.export({"x": "float64"}, "float64", capture_dtypes={"capture": "float64"}) + + def callback(x): + pytest.fail("callback executed") + + @blosc2.dsl_kernel + def external(x): + return callback(x) + + with pytest.raises(blosc2.PortableArtifactError, match="callback"): + external.export({"x": "float64"}, "float64") + + +def test_shadowed_numpy_alias_is_not_rewritten(native_artifacts): + @blosc2.dsl_kernel + def local_alias(x): + np = x + return np.sin(x) + + assert "np.sin" in local_alias.dsl_source + with pytest.raises(blosc2.PortableArtifactError): + local_alias.export({"x": "float64"}, "float64") + + +@pytest.mark.parametrize("jit", [False, None, True]) +def test_artifact_backend_and_concurrent_calls(native_artifacts, jit): + from concurrent.futures import ThreadPoolExecutor + + artifact = affine.export({"x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit is True and not kernel.has_jit: + pytest.skip("TCC backend not prepared on this host") + if jit is False: + assert not kernel.has_jit + arrays = [np.arange(600, dtype=np.float64) + i for i in range(4)] + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(lambda array: kernel.evaluate({"x": array}), arrays)) + for array, result in zip(arrays, results, strict=True): + np.testing.assert_array_equal(result, 2 * array - 1) + + +def test_capture_range_checks(native_artifacts): + kernel = blosc2.DSLKernel.from_source("def k(c):\n return c\n") + for value, dtype in [(2**63, "int64"), (2**31, "int32"), (2**53 + 1, "float64"), (1e100, "float32")]: + with pytest.raises(blosc2.PortableArtifactError): + kernel.export({}, dtype, constants={"c": value}, capture_dtypes={"c": dtype}) + + +def test_nan_payload_and_explicit_float_narrowing(native_artifacts): + author = blosc2.DSLKernel.from_source("def k(c):\n return c\n") + value = np.array([0x7FF8000000000042], dtype=np.uint64).view(np.float64)[0] + artifact = author.export({}, "float64", constants={"c": value}) + assert json.loads(artifact)["constants"][0]["value"] == "7ff8000000000042" + value = 1.1 + artifact = author.export({}, "float32", constants={"c": value}, capture_dtypes={"c": "float32"}) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + assert kernel.evaluate({}, shape=())[()] == np.float32(value) + + +def test_export_bindings_and_metadata_rejections(native_artifacts): + author = blosc2.DSLKernel.from_source("def k(x):\n return x\n") + with pytest.raises(blosc2.PortableArtifactError, match="exactly one"): + author.export({}, "float64") + with pytest.raises(blosc2.PortableArtifactError, match="exactly one"): + author.export({"x": "float64"}, "float64", constants={"x": 1.0}) + with pytest.raises(blosc2.PortableArtifactError) as error: + author.export({"x": "float64"}, "float64", metadata={"bad": "\x00"}) + assert error.value.status == "invalid_artifact" + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json(b"{\xff}") + assert error.value.status == "invalid_artifact" + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json("{\ud800}") + assert error.value.status == "invalid_artifact" + + +def test_unresolved_capture_has_location(native_artifacts): + @blosc2.dsl_kernel + def k(x): + return x + MISSING_PORTABLE_CAPTURE # noqa: F821 + + with pytest.raises(blosc2.PortableArtifactError) as error: + k.export({"x": "float64"}, "float64") + assert "MISSING_PORTABLE_CAPTURE" in str(error.value) + assert error.value.status == "invalid_source" + assert error.value.line > 0 + assert error.value.column > 0 + + +def test_binding_shapes_and_storage_adaptation(native_artifacts): + source = "def subtract(x, y):\n return x - y\n" + artifact = blosc2.DSLKernel.from_source(source).export({"y": "float64", "x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + x = np.arange(12, dtype=np.float64).reshape(3, 4)[:, ::2] + y = np.zeros((3, 2), dtype=">f8") + np.testing.assert_array_equal(kernel.evaluate({"y": y, "x": x}), x) + np.testing.assert_array_equal(kernel.evaluate({"y": y, "x": blosc2.asarray(x)}), x) + with pytest.raises(blosc2.PortableArtifactError, match="dtype"): + kernel.evaluate({"x": x.astype(np.float32), "y": y}) + with pytest.raises(blosc2.PortableArtifactError, match="shapes"): + kernel.evaluate({"x": x, "y": y.ravel()}) + with pytest.raises(blosc2.PortableArtifactError, match="Missing or extra"): + kernel.evaluate({"x": x}) + empty = np.empty((0, 3), dtype=np.float64) + assert kernel.evaluate({"y": empty, "x": empty}).shape == (0, 3) + + +def test_import_is_native_only(native_artifacts, monkeypatch): + if not ARTIFACTS.is_dir(): + pytest.skip("Native fixture checkout unavailable") + artifact = (ARTIFACTS / "affine.json").read_bytes() + + def forbidden(*args, **kwargs): + pytest.fail("Python source/JSON preparation was invoked") + + with monkeypatch.context() as context: + context.setattr(ast, "parse", forbidden) + context.setattr(json, "loads", forbidden) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + result = kernel.evaluate({"x": np.array([0.0, 1.0, 2.0, 3.0])}) + np.testing.assert_array_equal(result, [-1, 1, 3, 5]) + + +def test_export_runs_in_standalone_c(native_artifacts, tmp_path): + runner = Path( + os.environ.get( + "MINIEXPR_ARTIFACT_RUNNER", CORPUS.parents[1] / "build-portable/tests/portable_artifact_runner" + ) + ) + if not runner.is_file(): + pytest.skip("Set MINIEXPR_ARTIFACT_RUNNER to the standalone native runner") + artifact = tmp_path / "export.json" + artifact.write_text(affine.export({"x": "float64"}, "float64")) + result = subprocess.run([str(runner), str(artifact), "off"], check=True, capture_output=True, text=True) + assert result.stdout.splitlines() == ["jit=0", "-1", "1", "3", "5"] + # A fresh interpreter imports only blosc2, never this authoring test module. + program = "import pathlib, blosc2, numpy as np; " + program += ( + f"k = blosc2.PortableKernel.from_json(pathlib.Path({str(artifact)!r}).read_bytes(), jit=False); " + ) + program += "print(k.evaluate({'x': np.array([0., 1., 2., 3.])}).tolist())" + result = subprocess.run([sys.executable, "-c", program], check=True, capture_output=True, text=True) + assert result.stdout.strip() == "[-1.0, 1.0, 3.0, 5.0]" + + +def test_native_diagnostics_and_runtime_errors(native_artifacts): + artifact = affine.export({"x": "float64"}, "float64") + changed = artifact.replace('"schema_version":"0.1"', '"schema_version":"99"') + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json(changed, jit=False) + assert error.value.status == "unsupported_requirement" + changed = artifact.replace('"schema_version":', '"schema_version":"0.1","schema_version":') + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json(changed, jit=False) + assert error.value.status == "invalid_artifact" + source = "def partial(x):\n if x > 0:\n return x\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + with pytest.raises(blosc2.PortableArtifactError) as error: + kernel.evaluate({"x": np.array([-1.0])}) + assert error.value.status == "evaluation_error" + assert error.value.native_status != 0 + assert kernel.evaluate({"x": np.empty(0, dtype=np.float64)}).size == 0 + + +def test_import_missing_native_support(monkeypatch): + from blosc2 import blosc2_ext + + monkeypatch.delattr(blosc2_ext, "PortableArtifactHandle", raising=False) + with pytest.raises(NotImplementedError, match="artifact support"): + blosc2.PortableKernel.from_json("{}") + + +def test_optional_adapter_availability(): + from blosc2 import blosc2_ext + + available = getattr(blosc2_ext, "portable_artifact_available", lambda: False)() + if available: + with pytest.raises(blosc2.PortableArtifactError, match="invalid_artifact"): + blosc2.PortableKernel.from_json("{}", jit=False) + else: + with pytest.raises(NotImplementedError, match="artifact support"): + blosc2.PortableKernel.from_json("{}", jit=False) From eec381af0927a1e6bad73edddbe12d42993daa2b Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 17:32:14 +0200 Subject: [PATCH 06/30] Extend portable numeric conformance and track JIT gaps --- plans/portable-DSLs-first-version.md | 17 ++++++++++++ tests/test_dsl_portable.py | 39 ++++++++++++++++++++-------- tests/test_portable_artifact.py | 27 +++++++++++++++++++ 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 01bbef49e..5f3867ce8 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -112,6 +112,23 @@ without interpreter retry; generic backend failures retain existing fallback. This advances P2 coverage but does not freeze overflow, general promotions/conversions, or global transcendental accuracy rules. +The numeric stabilization slice adds 25 shared exact output-conversion fixtures +(the full supported 5×5 dtype matrix), including representable cast boundaries, +int64 rounding, signed zero, subnormals, and non-finite identity/truth values. +Native integer-literal validation now checks normalized digits exactly so +`2**53 + 1` cannot bypass the profile limit through double rounding. +The audit found unresolved integer-valued division promotion and float32 math +intermediate/condition discrepancies across JIT and linked interpreter paths. +Shared reproducers and strict expected-failure artifact tests preserve the gaps; +they block freezing rather than redefining portable semantics. Details are in +`../miniexpr/doc/dsl-spec/numeric-audit-0.1.md`. Native publication, dependency +integration, and the rest of the semantic matrix remain pending. + +This slice passed 328 focused Python tests with 10 strict expected JIT failures, +202 native portable/profile checks in the TCC-enabled build, and 55 selected +interpreter-only portable/profile/artifact checks. Ruff and whitespace checks +passed, and the native builds emitted no new warnings. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 217c426fa..c6e22d1e7 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -43,6 +43,12 @@ "missing_return_hybrid", "missing_return_hybrid_ok", ] +TYPES = ("bool", "int32", "int64", "float32", "float64") +CASES += [f"convert_{input_type}_{output_type}" for input_type in TYPES for output_type in TYPES] + + +def corpus_source(case): + return CORPUS / ("identity.dsl" if case.startswith("convert_") else f"{case}.dsl") def typed_values(values, dtype): @@ -50,13 +56,14 @@ def typed_values(values, dtype): return np.array([int(value) for value in values] if dtype == np.dtype("bool") else values, dtype=dtype) -def assert_typed_result(actual, expected): +def assert_typed_result(actual, expected, *, exact=False): assert actual.dtype == expected.dtype - if expected.dtype.kind in "biu": + if expected.dtype.kind in "biu" or exact: np.testing.assert_array_equal(actual, expected) - else: - tolerance = 1e-6 if expected.dtype == np.dtype("float32") else 1e-12 - np.testing.assert_allclose(actual, expected, rtol=tolerance, atol=tolerance, equal_nan=True) + if expected.dtype.kind == "f": + if not exact: + tolerance = 1e-6 if expected.dtype == np.dtype("float32") else 1e-12 + np.testing.assert_allclose(actual, expected, rtol=tolerance, atol=tolerance, equal_nan=True) zeros = expected == 0 np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) @@ -66,7 +73,7 @@ def assert_typed_result(actual, expected): def test_native_source_corpus(case, jit): if not CORPUS.is_dir(): pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") - source = (CORPUS / f"{case}.dsl").read_text() + source = corpus_source(case).read_text() words = (CORPUS / f"{case}.txt").read_text().split() outcome, input_type, output_type = words[:3] count, nvars = map(int, words[3:5]) @@ -77,9 +84,9 @@ def test_native_source_corpus(case, jit): kernel = DSLKernel.from_source(source) by_name = {name: typed_values(rows[:, index], input_dtype) for index, name in enumerate(names)} inputs = [by_name[name] for name in kernel.input_names] - if outcome == "ok": + if outcome in {"ok", "ok_exact"}: result = blosc2.lazyudf(kernel, inputs, dtype=output_dtype, jit=jit)[:] - assert_typed_result(result, expected) + assert_typed_result(result, expected, exact=outcome == "ok_exact") else: with pytest.raises((NotImplementedError, RuntimeError)): blosc2.lazyudf(kernel, inputs, dtype=output_dtype, jit=jit)[:] @@ -91,7 +98,7 @@ def test_native_source_corpus(case, jit): completed = subprocess.run( [ runner, - str(CORPUS / f"{case}.dsl"), + str(corpus_source(case)), str(CORPUS / f"{case}.txt"), "on" if jit else "off", ], @@ -107,7 +114,9 @@ def test_native_source_corpus(case, jit): if outcome == "eval_error": assert lines[1:] == [outcome] else: - assert_typed_result(typed_values(lines[1:], output_dtype), expected) + assert_typed_result( + typed_values(lines[1:], output_dtype), expected, exact=outcome == "ok_exact" + ) def test_from_source_never_executes_python(tmp_path): @@ -167,7 +176,7 @@ def test_portable_profile_corpus(portable_validator, case): outcome, input_type, output_type = words[:3] nvars = int(words[4]) inputs = dict.fromkeys(words[5 : 5 + nvars], input_type) - info = portable_validator((CORPUS / f"{case}.dsl").read_text(), inputs, output_type) + info = portable_validator(corpus_source(case).read_text(), inputs, output_type) assert info["valid"] == (outcome != "compile_error"), info @@ -243,3 +252,11 @@ def test_portable_profile_missing_native_support(monkeypatch): def test_portable_profile_rejects_nul(source, inputs, version): with pytest.raises(ValueError, match="NUL"): blosc2.validate_portable_dsl(source, inputs, "float64", language_version=version) + + +@pytest.mark.parametrize("literal", ["9007199254740993", "9_007_199_254_740_993", "0x20000000000001"]) +def test_portable_integer_literal_bound_is_exact(portable_validator, literal): + info = portable_validator(f"def k(x):\n return x + {literal}\n", {"x": "int64"}, "int64") + assert not info["valid"] + assert info["status"] == "unsupported_feature" + assert "portable range" in info["error"] diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 1e35492ef..ddde6967c 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -349,3 +349,30 @@ def test_optional_adapter_availability(): else: with pytest.raises(NotImplementedError, match="artifact support"): blosc2.PortableKernel.from_json("{}", jit=False) + + +@pytest.mark.parametrize( + "case", ["division_int", "division_bool", "division_literals", "math_widen", "math_condition"] +) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +def test_numeric_audit_gaps(native_artifacts, case, compiler, jit, request): + """Expected JIT gaps remain visible and block freezing, not silently standardized.""" + audit = CORPUS / "audit" + if not audit.is_dir(): + pytest.skip("Native audit fixtures unavailable") + words = (audit / f"{case}.txt").read_text().split() + input_dtype, output_dtype = words[1:3] + rows = np.array(words[6:]).reshape(int(words[3]), 2) + source = (audit / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit and not kernel.has_jit: + pytest.skip(f"{compiler} backend unavailable") + actual = kernel.evaluate({"x": np.array(rows[:, 0], dtype=input_dtype)}) + expected = np.array(rows[:, 1], dtype=output_dtype) + if jit: + # Add the marker only after successful compilation/backend preparation and + # execution, so unrelated setup/runtime failures cannot hide as xfails. + request.node.add_marker(pytest.mark.xfail(strict=True, reason=f"Open typed JIT codegen gap: {case}")) + np.testing.assert_array_equal(actual, expected) From 281ebe53e6deb68ff569695d16638634550fa312 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 17:57:06 +0200 Subject: [PATCH 07/30] Verify typed portable division and interpreter fallback --- plans/portable-DSLs-first-version.md | 19 +++++++++ tests/test_dsl_portable.py | 7 ++++ tests/test_portable_artifact.py | 61 ++++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 5f3867ce8..71135b19b 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -129,6 +129,25 @@ This slice passed 328 focused Python tests with 10 strict expected JIT failures, interpreter-only portable/profile/artifact checks. Ruff and whitespace checks passed, and the native builds emitted no new warnings. +The next slice corrects the first division-promotion cases using C arithmetic +rendered from the native compiled expression tree. Seven exact shared fixtures +cover leaf casts, literal folding, locals, loop indices, float32 rounding, and +branch conditions under interpreter/TCC/CC execution. Typed text participates in +the IR fingerprint and code-generation cache version 14. Text-only hybrid plans +cannot bypass the correction. Unsupported typed contexts conservatively retain +interpreter execution; native-only artifact import and compiler precedence are +unchanged. The six division expected failures are removed; four float32 math +expected failures remain. Broader nested-cast probes exposed additional native +evaluator dispatch anomalies and an unisolated abort, recorded in the numeric +audit as separate blockers rather than promoted conformance claims. + +Division-slice validation passed 377 focused Python tests with four remaining +math expected failures, all 272 native tests in the TCC-enabled build, and 61 +selected interpreter-only portable/profile/artifact/IR/codegen tests. Required-JIT +division fixtures passed under both TCC and CC. Ruff and whitespace checks passed; +native builds emitted no new warnings. The nested-cast reproducer is deliberately +not part of the passing execution suite pending a native evaluator investigation. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index c6e22d1e7..f45848b4d 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -32,6 +32,13 @@ "boolean_results", "cast_int", "math", + "division_int", + "division_bool", + "division_literals", + "division_locals", + "division_loop", + "division_float32", + "division_condition", "unknown_name", "invalid_index", "zero_step", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index ddde6967c..da65ce07d 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -351,9 +351,7 @@ def test_optional_adapter_availability(): blosc2.PortableKernel.from_json("{}", jit=False) -@pytest.mark.parametrize( - "case", ["division_int", "division_bool", "division_literals", "math_widen", "math_condition"] -) +@pytest.mark.parametrize("case", ["math_widen", "math_condition"]) @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) def test_numeric_audit_gaps(native_artifacts, case, compiler, jit, request): @@ -376,3 +374,60 @@ def test_numeric_audit_gaps(native_artifacts, case, compiler, jit, request): # execution, so unrelated setup/runtime failures cannot hide as xfails. request.node.add_marker(pytest.mark.xfail(strict=True, reason=f"Open typed JIT codegen gap: {case}")) np.testing.assert_array_equal(actual, expected) + + +@pytest.mark.parametrize( + "case", + [ + "division_int", + "division_bool", + "division_literals", + "division_locals", + "division_loop", + "division_float32", + "division_condition", + ], +) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +def test_typed_division_artifact(native_artifacts, case, compiler, jit): + if not (CORPUS / f"{case}.txt").is_file(): + pytest.skip("Native division fixtures unavailable") + if jit: + control = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n return x\n") + control = blosc2.PortableKernel.from_json(control.export({"x": "float64"}, "float64"), jit=True) + if not control.has_jit: + pytest.skip(f"{compiler} backend unavailable") + words = (CORPUS / f"{case}.txt").read_text().split() + input_dtype, output_dtype = words[1:3] + rows = np.array(words[6:]).reshape(int(words[3]), 2) + source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert kernel.has_jit == jit + np.testing.assert_array_equal( + kernel.evaluate({"x": np.array(rows[:, 0], dtype=input_dtype)}), + np.array(rows[:, 1], dtype=output_dtype), + ) + + +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize( + ("body", "output_dtype", "expected"), + [ + ("return int(x / 2) / 2", "float64", [-0.5, 0, 0, 0, 0.5]), + ("return int(x) / 2", "int64", [-1, 0, 0, 0, 1]), + ("if (int(x) / 2) > 0:\n return 1.0\n return 0.0", "float64", [0, 0, 0, 1, 1]), + ], +) +def test_unsupported_division_lowering_uses_interpreter( + native_artifacts, compiler, body, output_dtype, expected +): + source = f"# me:compiler={compiler}\ndef k(x):\n {body}\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=True) + assert not kernel.has_jit + np.testing.assert_array_equal( + kernel.evaluate({"x": np.array([-3.75, -1.75, 0, 1.75, 3.75])}), + np.array(expected, dtype=output_dtype), + ) From 1b9c4780f71acb8070aecc967cd4a62ac797c21b Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 18:14:13 +0200 Subject: [PATCH 08/30] Test portable nested conversion buffer safety --- plans/portable-DSLs-first-version.md | 17 +++++++++++++++-- tests/test_portable_artifact.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 71135b19b..a9336cd6a 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -145,8 +145,21 @@ Division-slice validation passed 377 focused Python tests with four remaining math expected failures, all 272 native tests in the TCC-enabled build, and 61 selected interpreter-only portable/profile/artifact/IR/codegen tests. Required-JIT division fixtures passed under both TCC and CC. Ruff and whitespace checks passed; -native builds emitted no new warnings. The nested-cast reproducer is deliberately -not part of the passing execution suite pending a native evaluator investigation. +native builds emitted no new warnings. At that stage the nested-cast reproducer +was excluded from execution conformance pending a native evaluator investigation. + +The following memory-safety slice isolated that abort with AddressSanitizer: +nested conversion code selected a float64 writer for a float32-sized scratch +buffer. The writer now matches the actual typed evaluator's output representation +while retaining source-dtype evaluation. Exact interpreter fixtures and native/ +Python tests cover float32/float64 input/output pairs, counts 1/5/257, repeated +evaluation, and requested-JIT fallback. Sanitizer validation also corrected an +existing mixed-type test's undersized output allocation (int32 + float32 infers +float64). All 217 sanitizer-build tests, all 274 regular native tests, and 401 +focused Python tests passed, with four existing math expected failures. Ruff and +whitespace checks passed; builds emitted no new warnings. This fixes the confirmed +buffer overflow, not broader nested-cast promotion or float32 math semantics; +those still block freezing 0.1. The native dependency pin remains unchanged. ## 1. Goal and scope diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index da65ce07d..907c142e3 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -431,3 +431,19 @@ def test_unsupported_division_lowering_uses_interpreter( kernel.evaluate({"x": np.array([-3.75, -1.75, 0, 1.75, 3.75])}), np.array(expected, dtype=output_dtype), ) + + +@pytest.mark.parametrize("input_dtype", ["float32", "float64"]) +@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) +@pytest.mark.parametrize("count", [1, 5, 257]) +@pytest.mark.parametrize("jit", [False, True]) +def test_nested_conversion_buffer_width(native_artifacts, input_dtype, output_dtype, count, jit): + source = "def k(x):\n return float(int(x) / 2)\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + # Nested casts remain intentionally outside the typed JIT arithmetic slice. + assert not kernel.has_jit + values = np.resize(np.array([-1.75, -0.25, 0.25, 1.75, 4.75], dtype=input_dtype), count) + expected = np.resize(np.array([-0.5, 0, 0, 0.5, 2], dtype=output_dtype), count) + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) From 5ca5bd396e6fb5ef7c64140081397150f05eec6c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 18:46:51 +0200 Subject: [PATCH 09/30] Verify portable cast argument value semantics --- plans/portable-DSLs-first-version.md | 14 ++++++++++ tests/test_dsl_portable.py | 5 ++++ tests/test_portable_artifact.py | 42 ++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index a9336cd6a..a7ec0e166 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -161,6 +161,20 @@ whitespace checks passed; builds emitted no new warnings. This fixes the confirm buffer overflow, not broader nested-cast promotion or float32 math semantics; those still block freezing 0.1. The native dependency pin remains unchanged. +The next slice corrects `int()`/`bool()` argument value semantics: evaluate in the +argument's compiled dtype before truncation/nonzero truth, instead of promoting +variables or recomputing arithmetic in the cast result dtype. Integer casts +preserve int64 values beyond `2**53` without a double round trip; Boolean casts +retain non-finite and subnormal truth. Five exact shared fixtures require real +TCC/CC execution as well as interpreter agreement; a separate division audit +case checks conservative interpreter fallback. Native repeated-evaluation tests +cover counts 1/5/257 and floating widths, while Python tests cover all five output +dtypes for value casts. All 295 regular native tests, 233 AddressSanitizer tests, +and 484 focused Python tests passed, with four existing math expected failures. +No new build warnings were emitted. General `float()`/division promotion, +out-of-domain integer casts, and float32 math semantics remain unfinished; the +profile is still experimental and the dependency pin remains unchanged. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index f45848b4d..46db72217 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -39,6 +39,11 @@ "division_loop", "division_float32", "division_condition", + "cast_argument_int", + "cast_argument_bool", + "cast_argument_nested", + "cast_integer_exact", + "cast_nonfinite_truth", "unknown_name", "invalid_index", "zero_step", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 907c142e3..a387d8139 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -417,6 +417,7 @@ def test_typed_division_artifact(native_artifacts, case, compiler, jit): [ ("return int(x / 2) / 2", "float64", [-0.5, 0, 0, 0, 0.5]), ("return int(x) / 2", "int64", [-1, 0, 0, 0, 1]), + ("return int(x + 0.25) / 2", "float64", [-1.5, -0.5, 0, 1, 2]), ("if (int(x) / 2) > 0:\n return 1.0\n return 0.0", "float64", [0, 0, 0, 1, 1]), ], ) @@ -447,3 +448,44 @@ def test_nested_conversion_buffer_width(native_artifacts, input_dtype, output_dt expected = np.resize(np.array([-0.5, 0, 0, 0.5, 2], dtype=output_dtype), count) for _ in range(2): np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("int(x + 0.25)", [-1, 0, 0, 2, 5]), + ("bool(x + 0.25)", [1, 0, 1, 1, 1]), + ("bool(int(x + 0.25))", [1, 0, 0, 1, 1]), + ], +) +@pytest.mark.parametrize("input_dtype", ["float32", "float64"]) +@pytest.mark.parametrize("output_dtype", ["bool", "int32", "int64", "float32", "float64"]) +@pytest.mark.parametrize("jit", [False, True]) +def test_value_cast_argument_semantics( + native_artifacts, expression, expected, input_dtype, output_dtype, jit +): + source = f"# me:compiler=tcc\ndef k(x):\n return {expression}\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + values = np.array([-1.75, -0.25, 0.25, 1.75, 4.75], dtype=input_dtype) + np.testing.assert_array_equal(kernel.evaluate({"x": values}), np.array(expected, dtype=output_dtype)) + + +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("expression", ["int(x)", "bool(x)"]) +def test_value_cast_large_integer(native_artifacts, jit, expression): + source = f"# me:compiler=tcc\ndef k(x):\n return {expression}\n" + output_dtype = "int64" if expression == "int(x)" else "bool" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + values = np.array([-(2**53 + 1), 0, 2**53 + 1, 2**63 - 1], dtype="int64") + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values.astype(output_dtype)) + + +@pytest.mark.parametrize("jit", [False, True]) +def test_value_cast_nonfinite_truth(native_artifacts, jit): + source = "# me:compiler=tcc\ndef k(x):\n return bool(x)\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, "bool") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + values = np.array([-0.0, 0.0, np.nan, np.inf, -np.inf, 5e-324]) + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values.astype("bool")) From fca1ad316e279626342f0f18df08ee62a394d195 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 19:11:15 +0200 Subject: [PATCH 10/30] Certify portable float32 leaf math and branch rounding --- plans/portable-DSLs-first-version.md | 15 ++++++++++++++ tests/test_dsl_portable.py | 3 +++ tests/test_portable_artifact.py | 29 +++++++++++++--------------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index a7ec0e166..f185d5b45 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -175,6 +175,21 @@ No new build warnings were emitted. General `float()`/division promotion, out-of-domain integer casts, and float32 math semantics remain unfinished; the profile is still experimental and the dependency pin remains unchanged. +The following float32 math slice resolves the two remaining expected-failure +reproducers for leaf `sin` widening and `cos` branch conditions. Strict leaf +float32 sin/cos uses scalar float math at every count, avoiding the one-ULP SIMD +variation observed at count 257; comparison evaluation no longer pre-promotes +variables before consuming operand values. Owned typed JIT math text preserves +float rounding before widening/comparison and bypasses text-only hybrid plans; +it enters the IR fingerprint, with code-generation cache version 15. The two +audit fixtures are promoted to interpreter/TCC/CC conformance, with a local-value +widening fixture and native/Python repeated-execution checks at counts 1/2/257. +All 307 regular native tests, 242 AddressSanitizer tests, and 525 focused Python +tests passed without expected failures or new build warnings. General nested +math, other functions, inexact comparison constants, broader promotion/cast +domains, and cross-platform math accuracy still block freezing the profile; +dependency integration remains pending. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 46db72217..a07a3b7b3 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -44,6 +44,9 @@ "cast_argument_nested", "cast_integer_exact", "cast_nonfinite_truth", + "math_widen", + "math_condition", + "math_local_widen", "unknown_name", "invalid_index", "zero_step", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index a387d8139..2db75a54f 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -351,29 +351,26 @@ def test_optional_adapter_availability(): blosc2.PortableKernel.from_json("{}", jit=False) -@pytest.mark.parametrize("case", ["math_widen", "math_condition"]) +@pytest.mark.parametrize("case", ["math_widen", "math_condition", "math_local_widen"]) @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) -def test_numeric_audit_gaps(native_artifacts, case, compiler, jit, request): - """Expected JIT gaps remain visible and block freezing, not silently standardized.""" - audit = CORPUS / "audit" - if not audit.is_dir(): - pytest.skip("Native audit fixtures unavailable") - words = (audit / f"{case}.txt").read_text().split() +@pytest.mark.parametrize("count", [1, 2, 257]) +def test_float32_leaf_math(native_artifacts, case, compiler, jit, count): + """Leaf math rounds in float32 before widening or comparing.""" + if not CORPUS.is_dir(): + pytest.skip("Native fixtures unavailable") + words = (CORPUS / f"{case}.txt").read_text().split() input_dtype, output_dtype = words[1:3] rows = np.array(words[6:]).reshape(int(words[3]), 2) - source = (audit / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit and not kernel.has_jit: - pytest.skip(f"{compiler} backend unavailable") - actual = kernel.evaluate({"x": np.array(rows[:, 0], dtype=input_dtype)}) - expected = np.array(rows[:, 1], dtype=output_dtype) if jit: - # Add the marker only after successful compilation/backend preparation and - # execution, so unrelated setup/runtime failures cannot hide as xfails. - request.node.add_marker(pytest.mark.xfail(strict=True, reason=f"Open typed JIT codegen gap: {case}")) - np.testing.assert_array_equal(actual, expected) + assert kernel.has_jit + values = np.resize(np.array(rows[:, 0], dtype=input_dtype), count) + expected = np.resize(np.array(rows[:, 1], dtype=output_dtype), count) + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) @pytest.mark.parametrize( From e574d7fdb92ae641ae1a0191487124e0e0d81cd2 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 19:33:30 +0200 Subject: [PATCH 11/30] Verify portable floating literal and arithmetic rounding --- plans/portable-DSLs-first-version.md | 16 +++++++++++ tests/test_dsl_portable.py | 5 ++++ tests/test_portable_artifact.py | 41 ++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index f185d5b45..9a6e18916 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -190,6 +190,22 @@ math, other functions, inexact comparison constants, broader promotion/cast domains, and cross-platform math accuracy still block freezing the profile; dependency integration remains pending. +The next literal/promotion slice corrects pure float32 arithmetic: C double +literals must not bypass native operand or intermediate rounding for `+`, `-`, +`*`, and unary negation. Five shared exact fixtures cover exact/decimal literal +rounding, subtraction, locals, and differing floating output context. Literal +typing is contextual: a float64 requested output can make literal-containing +arithmetic compute in float64, rather than just widen a float32 result. The new +owned lowering targets float32 computation only, bypasses text-only hybrid plans, +enters the IR fingerprint, and uses code-generation cache version 16. Ordinary +float64 hybrid branch optimizations are preserved. Native/Python tests check +counts 1/5/257, repeated execution, both floating output widths, and scalar +addition/subtraction/multiplication. All 328 regular native tests, 258 +AddressSanitizer tests, and 632 focused Python tests passed without expected +failures or new build warnings. Calls/casts, mixed intermediate dtypes, integral/ +Boolean arithmetic, overflow, and the complete promotion contract remain audit +work; the profile is still experimental and dependency integration is pending. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index a07a3b7b3..a1c9ed8f4 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -47,6 +47,11 @@ "math_widen", "math_condition", "math_local_widen", + "arithmetic_round", + "arithmetic_decimal", + "arithmetic_scalar_sub", + "arithmetic_local_round", + "arithmetic_widen", "unknown_name", "invalid_index", "zero_step", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 2db75a54f..ba3c31ea5 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -383,13 +383,18 @@ def test_float32_leaf_math(native_artifacts, case, compiler, jit, count): "division_loop", "division_float32", "division_condition", + "arithmetic_round", + "arithmetic_decimal", + "arithmetic_scalar_sub", + "arithmetic_local_round", + "arithmetic_widen", ], ) @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) -def test_typed_division_artifact(native_artifacts, case, compiler, jit): +def test_typed_arithmetic_artifact(native_artifacts, case, compiler, jit): if not (CORPUS / f"{case}.txt").is_file(): - pytest.skip("Native division fixtures unavailable") + pytest.skip("Native arithmetic fixtures unavailable") if jit: control = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n return x\n") control = blosc2.PortableKernel.from_json(control.export({"x": "float64"}, "float64"), jit=True) @@ -408,6 +413,38 @@ def test_typed_division_artifact(native_artifacts, case, compiler, jit): ) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("count", [1, 5, 257]) +@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) +@pytest.mark.parametrize("operation", ["add", "subtract", "multiply"]) +def test_float_arithmetic_constant_rounding(native_artifacts, compiler, jit, count, output_dtype, operation): + expressions = { + "add": "(x + 0.1) - x", + "subtract": "x - 0.1", + "multiply": "(x * 0.1) - x", + } + source = f"# me:compiler={compiler}\n# me:fp=strict\ndef k(x):\n return {expressions[operation]}\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + values = np.resize(np.array([0.1, -1, 0, 1, 16777216], dtype="float32"), count) + # These expressions contain a floating literal: native compilation types + # that constant from the requested floating output context. Each operation + # then rounds its operands/result in that context, not C's double literal type. + arithmetic_values = values.astype(output_dtype) + constant = np.array(0.1, dtype=output_dtype) + if operation == "add": + expected = (arithmetic_values + constant) - arithmetic_values + elif operation == "subtract": + expected = arithmetic_values - constant + else: + expected = (arithmetic_values * constant) - arithmetic_values + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + + @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize( ("body", "output_dtype", "expected"), From f7af684b199fa5d884ae8c50e2d491064f7fca76 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 19:44:04 +0200 Subject: [PATCH 12/30] Verify portable loop-cap policies and error handling --- plans/portable-DSLs-first-version.md | 16 ++++ tests/test_dsl_portable.py | 8 +- tests/test_portable_artifact.py | 135 +++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 1 deletion(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 9a6e18916..d83bfe646 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -206,6 +206,22 @@ failures or new build warnings. Calls/casts, mixed intermediate dtypes, integral Boolean arithmetic, overflow, and the complete promotion contract remain audit work; the profile is still experimental and dependency integration is pending. +The while-loop execution-policy slice fixes missing JIT cap enforcement. Scoped +body-entry counters preserve exact-limit exits, `continue`, nested/re-entered +loops, and condition-prefix lowering. The normalized host cap enters the IR +fingerprint, with code-generation cache version 17; a changed policy after +compilation bypasses both JIT evaluation paths for current-policy interpreter +execution. Semantic cap errors propagate without interpreter retry and release +hybrid temporaries. Four shared fixtures and expanded native/Python tests cover +small caps, disabled/invalid policies, counts 1/5/257, repeated/empty execution, +cache isolation, policy changes, and boundary control flow. A separate +interpreter mixed-lane chained-while discrepancy is preserved in +`audit/while_cap_chain` and a strict expected-failure test; it blocks freezing +the profile alongside the remaining numeric domains and dependency integration. +Validation passed all 344 regular native tests, 270 AddressSanitizer tests, and +727 focused Python tests with one strict chained-while expected failure. Ruff +and whitespace checks passed; native builds emitted no new warnings. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index a1c9ed8f4..e7411675a 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -52,6 +52,10 @@ "arithmetic_scalar_sub", "arithmetic_local_round", "arithmetic_widen", + "while_cap_exact", + "while_cap_exceeded", + "while_cap_continue", + "while_cap_hybrid", "unknown_name", "invalid_index", "zero_step", @@ -90,7 +94,9 @@ def assert_typed_result(actual, expected, *, exact=False): @pytest.mark.parametrize("case", CASES) @pytest.mark.parametrize("jit", [False, True]) -def test_native_source_corpus(case, jit): +def test_native_source_corpus(case, jit, monkeypatch): + if case.startswith("while_cap_"): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") if not CORPUS.is_dir(): pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") source = corpus_source(case).read_text() diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index ba3c31ea5..25a7a2072 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -445,6 +445,141 @@ def test_float_arithmetic_constant_rounding(native_artifacts, compiler, jit, cou np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("count", [1, 5, 257]) +@pytest.mark.parametrize("cap", ["3", "0", "-1", "invalid"]) +def test_while_cap_policy(native_artifacts, monkeypatch, compiler, jit, count, cap): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", cap) + source = f"# me:compiler={compiler}\ndef k(x):\n n = 0\n while n < x:\n n = n + 1\n return n\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + for target in [0, 2, 3, 4]: + values = np.full(count, target, dtype="int64") + if cap == "3" and target == 4: + with pytest.raises(blosc2.PortableArtifactError) as error: + kernel.evaluate({"x": values}) + assert error.value.status == "evaluation_error" + else: + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values) + # Failed output contents are unspecified; the same prepared handle remains usable. + values = np.full(count, 2, dtype="int64") + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values) + assert kernel.evaluate({"x": np.empty(0, dtype="int64")}).size == 0 + + +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("compile_cap", ["0", "3"]) +def test_while_cap_change_after_loading(native_artifacts, monkeypatch, compiler, jit, compile_cap): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", compile_cap) + source = f"# me:compiler={compiler}\ndef k(x):\n n = 0\n while n < x:\n n = n + 1\n return n\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + values = np.array([4], dtype="int64") + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "2") + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + kernel.evaluate({"x": values}) + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "4") + np.testing.assert_array_equal(kernel.evaluate({"x": values}), values) + # The identical source loaded under a different cap must not reuse code + # compiled with the old cap from the shared JIT cache. + new_kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert new_kernel.has_jit + np.testing.assert_array_equal(new_kernel.evaluate({"x": values}), values) + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + kernel.evaluate({"x": values}) + + +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +def test_while_cap_hybrid_cleanup(native_artifacts, monkeypatch, compiler): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + source = ( + f"# me:compiler={compiler}\ndef k(x):\n value = sin(x)\n n = 0\n" + " while n < x:\n n = n + 1\n return value\n" + ) + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=True) + assert kernel.has_jit + for _ in range(4): + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + kernel.evaluate({"x": np.full(257, 4.0)}) + np.testing.assert_array_equal(kernel.evaluate({"x": np.zeros(257)}), np.zeros(257)) + + +@pytest.mark.parametrize("jit", [False, True]) +def test_while_chain_mixed_lane_audit(native_artifacts, monkeypatch, jit, request): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + source = (CORPUS / "audit" / "while_cap_chain.dsl").read_text() + artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + else: + # Preparation must succeed before marking this known interpreter error. + request.node.add_marker( + pytest.mark.xfail( + strict=True, + raises=blosc2.PortableArtifactError, + reason="Open interpreter chained-while mixed-lane condition gap", + ) + ) + values = np.array([0, 2, 3], dtype="int64") + try: + actual = kernel.evaluate({"x": values}) + except blosc2.PortableArtifactError as error: + if error.status != "evaluation_error": + raise RuntimeError("Unexpected failure outside the chained-while audit gap") from error + raise + np.testing.assert_array_equal(actual, values) + + +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize( + ("body", "expected"), + [ + (" n = 0\n while 0 <= n < 3:\n n = n + 1\n return n\n", 3), + ( + " n = 0\n while x:\n n = n + 1\n if n == 3:\n break\n return n\n", + 3, + ), + ( + " n = 0\n while x:\n n = n + 1\n if n == 3:\n return n\n return n\n", + 3, + ), + ( + " total = 0\n for i in range(2):\n n = 0\n while n < 3:\n" + " n = n + 1\n total = total + 1\n return total\n", + 6, + ), + ( + " total = 0\n n = 0\n while n < 3:\n n = n + 1\n m = 0\n" + " while m < 3:\n m = m + 1\n total = total + 1\n return total\n", + 9, + ), + (" if x > 1:\n while x:\n pass\n return x\n", 1), + ], +) +def test_while_cap_control_flow(native_artifacts, monkeypatch, compiler, jit, body, expected): + monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") + artifact = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n{body}").export( + {"x": "int64"}, "int64" + ) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + np.testing.assert_array_equal( + kernel.evaluate({"x": np.ones(5, dtype="int64")}), np.full(5, expected, dtype="int64") + ) + + @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize( ("body", "output_dtype", "expected"), From b27849a931a75e659605c80bec5d8480e71fd733 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 19:53:49 +0200 Subject: [PATCH 13/30] Verify masked local and chained-loop lane semantics --- plans/portable-DSLs-first-version.md | 11 ++++++ tests/test_dsl_portable.py | 3 ++ tests/test_portable_artifact.py | 50 +++++++++++++++++----------- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index d83bfe646..c6b9050a0 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -222,6 +222,17 @@ Validation passed all 344 regular native tests, 270 AddressSanitizer tests, and 727 focused Python tests with one strict chained-while expected failure. Ruff and whitespace checks passed; native builds emitted no new warnings. +The subsequent masked-local correction resolves that chained-while discrepancy. +Scalar chain operands now read the active lane of local buffers rather than +assuming compile-time uniformity survives masked updates. Root reduction copies +broadcast their scalar results into full-width DSL destinations, preserving +existing nonportable reduction behavior. `while_cap_chain` is promoted to the +passing corpus, with numeric/Boolean masked-local fixtures and expanded lane, +dtype, count, and repeated-evaluation matrices. Validation passed 357 regular +native tests, 280 AddressSanitizer tests, and 895 focused Python tests without +expected failures or new native build warnings. The remaining numeric contracts +and publication/integration gates are unchanged. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index e7411675a..d31531996 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -56,6 +56,9 @@ "while_cap_exceeded", "while_cap_continue", "while_cap_hybrid", + "while_cap_chain", + "masked_local_chain", + "masked_bool_chain", "unknown_name", "invalid_index", "zero_step", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 25a7a2072..822b72359 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -513,31 +513,41 @@ def test_while_cap_hybrid_cleanup(native_artifacts, monkeypatch, compiler): np.testing.assert_array_equal(kernel.evaluate({"x": np.zeros(257)}), np.zeros(257)) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) -def test_while_chain_mixed_lane_audit(native_artifacts, monkeypatch, jit, request): +@pytest.mark.parametrize("dtype", ["int32", "int64", "float32", "float64"]) +@pytest.mark.parametrize("count", [1, 3, 257]) +@pytest.mark.parametrize("samples", [[0, 2, 3, -1], [3, 2, 0, -1]]) +def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dtype, count, samples): monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") - source = (CORPUS / "audit" / "while_cap_chain.dsl").read_text() - artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") + source = ( + (CORPUS / "while_cap_chain.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + ) + artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) if jit: assert kernel.has_jit - else: - # Preparation must succeed before marking this known interpreter error. - request.node.add_marker( - pytest.mark.xfail( - strict=True, - raises=blosc2.PortableArtifactError, - reason="Open interpreter chained-while mixed-lane condition gap", - ) - ) - values = np.array([0, 2, 3], dtype="int64") - try: - actual = kernel.evaluate({"x": values}) - except blosc2.PortableArtifactError as error: - if error.status != "evaluation_error": - raise RuntimeError("Unexpected failure outside the chained-while audit gap") from error - raise - np.testing.assert_array_equal(actual, values) + values = np.resize(np.array(samples, dtype=dtype), count) + expected = np.maximum(values, np.array(0, dtype=dtype)) + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + + +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("dtype", ["int32", "int64", "float32", "float64"]) +@pytest.mark.parametrize("count", [4, 257]) +@pytest.mark.parametrize("case", ["masked_local_chain", "masked_bool_chain"]) +def test_masked_local_constant_chain(native_artifacts, compiler, jit, dtype, count, case): + source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, "bool") + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + values = np.resize(np.array([0, 1, 2, -1], dtype=dtype), count) + expected = np.resize(np.array([False, True, True, case == "masked_bool_chain"]), count) + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) @pytest.mark.parametrize("compiler", ["tcc", "cc"]) From 3015a930faad5bbcda58fddefb64f8614ada7241 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 20:21:55 +0200 Subject: [PATCH 14/30] Verify portable float-cast arithmetic rounding --- plans/portable-DSLs-first-version.md | 13 ++++++++ tests/test_dsl_portable.py | 4 +++ tests/test_portable_artifact.py | 50 ++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index c6b9050a0..852fb1c77 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -233,6 +233,19 @@ native tests, 280 AddressSanitizer tests, and 895 focused Python tests without expected failures or new native build warnings. The remaining numeric contracts and publication/integration gates are unchanged. +The next numeric slice fixes float32 rounding lost around same-dtype `float()` +calls: `(float(x) + 1.0) - float(x)` and `float(x + 1.0) - float(x)` now return +the interpreter's zero at `x = 2**24`, instead of JIT's previous one. Typed pure +arithmetic admits only float32-result calls with float32-computed arguments, +recursing through the already supported arithmetic. Root calls, nested/repeated +calls, and local assignments preserve rounding; float64 output retains observed +contextual literal semantics. Mixed/integral casts and general calls are not +certified. Four shared fixtures and expanded scalar/vector matrices cover both +compilers, output widths, decimal literals, signed zero, subnormals, extrema, +infinities, and NaNs. Code-generation cache version is 18. Validation passed all +373 regular native tests, 292 AddressSanitizer tests, and 1019 focused Python +tests without expected failures or new native build warnings. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index d31531996..a6808afcb 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -52,6 +52,10 @@ "arithmetic_scalar_sub", "arithmetic_local_round", "arithmetic_widen", + "arithmetic_float_cast", + "arithmetic_float_nested", + "arithmetic_float_local", + "arithmetic_float_widen", "while_cap_exact", "while_cap_exceeded", "while_cap_continue", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 822b72359..1727600f7 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -351,6 +351,52 @@ def test_optional_adapter_availability(): blosc2.PortableKernel.from_json("{}", jit=False) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("count", [1, 10, 257]) +@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) +@pytest.mark.parametrize("operation", ["leaf", "nested", "repeated", "negation"]) +def test_float_cast_arithmetic_rounding(native_artifacts, compiler, jit, count, output_dtype, operation): + expressions = { + "leaf": "(float(x) + 0.1) - float(x)", + "nested": "float(x + 0.1) - float(x)", + "repeated": "float(float(x) + 0.1) - float(float(x))", + "negation": "float(-(x * 0.1))", + } + source = f"# me:compiler={compiler}\ndef k(x):\n return {expressions[operation]}\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + samples = np.array( + [ + 2**24, + -(2**24), + 0.0, + -0.0, + np.finfo("float32").smallest_subnormal, + np.finfo("float32").max, + np.inf, + -np.inf, + np.nan, + 0.1, + ], + dtype="float32", + ) + values = np.resize(samples, count) + computation = values.astype(output_dtype) + constant = np.array(0.1, dtype=output_dtype) + with np.errstate(all="ignore"): + expected = ( + -(computation * constant) if operation == "negation" else (computation + constant) - computation + ) + for _ in range(2): + actual = kernel.evaluate({"x": values}) + np.testing.assert_array_equal(actual, expected) + zeros = expected == 0 + np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) + + @pytest.mark.parametrize("case", ["math_widen", "math_condition", "math_local_widen"]) @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) @@ -388,6 +434,10 @@ def test_float32_leaf_math(native_artifacts, case, compiler, jit, count): "arithmetic_scalar_sub", "arithmetic_local_round", "arithmetic_widen", + "arithmetic_float_cast", + "arithmetic_float_nested", + "arithmetic_float_local", + "arithmetic_float_widen", ], ) @pytest.mark.parametrize("compiler", ["tcc", "cc"]) From 9aa3d545cd58b1c016c9aef06795d7563fcd360c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 20:32:31 +0200 Subject: [PATCH 15/30] Verify Boolean operands in portable floating arithmetic --- plans/portable-DSLs-first-version.md | 14 ++++++++ tests/test_dsl_portable.py | 5 +++ tests/test_portable_artifact.py | 51 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 852fb1c77..609bdda7d 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -246,6 +246,20 @@ infinities, and NaNs. Code-generation cache version is 18. Validation passed all 373 regular native tests, 292 AddressSanitizer tests, and 1019 focused Python tests without expected failures or new native build warnings. +The Boolean-operand floating-arithmetic slice corrects interpreter storage: +`bool(x) + bool(x)` with integral inputs and floating output now returns two +for nonzero input rather than writing an invalid Boolean byte before conversion. +The DSL preserves the requested floating computation dtype for Boolean-inferred +arithmetic, while casts still consume their arguments in their own dtype. +Typed unary negation casts before operating so false produces floating negative +zero, including top-level bool-assigned locals. Float64 lowering stays narrow +to preserve hybrid branch/select plans; cache version is 19. Five shared fixtures +and direct/local scalar/vector matrices passed 393 regular native tests, +307 AddressSanitizer tests, and 2014 focused Python tests without expected +failures or new native build warnings. `audit/bool_output_fraction` preserves an +unresolved Boolean-output arithmetic discrepancy, not a normative rule; that +domain and broader mixed-type promotion remain publication gates. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index a6808afcb..2c1e92795 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -56,6 +56,11 @@ "arithmetic_float_nested", "arithmetic_float_local", "arithmetic_float_widen", + "bool_float_sum", + "bool_float_neg", + "bool_float_product", + "bool_float_difference", + "bool_float_local_neg", "while_cap_exact", "while_cap_exceeded", "while_cap_continue", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 1727600f7..e5d757cc7 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -351,6 +351,52 @@ def test_optional_adapter_availability(): blosc2.PortableKernel.from_json("{}", jit=False) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("input_dtype", ["bool", "int32", "int64", "float32", "float64"]) +@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) +@pytest.mark.parametrize("count", [1, 10, 257]) +@pytest.mark.parametrize("operation", ["sum", "difference", "product", "negation"]) +@pytest.mark.parametrize("local", [False, True]) +def test_bool_cast_floating_arithmetic( + native_artifacts, compiler, jit, input_dtype, output_dtype, count, operation, local +): + expressions = { + "sum": "bool(x) + bool(x)", + "difference": "bool(x) - bool(x)", + "product": "bool(x) * bool(x)", + "negation": "-bool(x)", + } + expression = expressions[operation] + body = "" + if local: + expression = expression.replace("bool(x)", "truth") + body = " truth = bool(x)\n" + source = f"# me:compiler={compiler}\ndef k(x):\n{body} return {expression}\n" + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + samples = [0, 1, -1, 2] + if input_dtype == "int64": + samples += [2**53 + 1, -(2**53 + 1)] + elif input_dtype.startswith("float"): + samples += [-0.0, np.finfo(input_dtype).smallest_subnormal, np.inf, -np.inf, np.nan] + values = np.resize(np.array(samples, dtype=input_dtype), count) + truth = (values != 0).astype(output_dtype) + expected = { + "sum": truth + truth, + "difference": truth - truth, + "product": truth * truth, + "negation": -truth, + }[operation] + for _ in range(2): + actual = kernel.evaluate({"x": values}) + np.testing.assert_array_equal(actual, expected) + zeros = expected == 0 + np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) + + @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) @pytest.mark.parametrize("count", [1, 10, 257]) @@ -438,6 +484,11 @@ def test_float32_leaf_math(native_artifacts, case, compiler, jit, count): "arithmetic_float_nested", "arithmetic_float_local", "arithmetic_float_widen", + "bool_float_sum", + "bool_float_neg", + "bool_float_product", + "bool_float_difference", + "bool_float_local_neg", ], ) @pytest.mark.parametrize("compiler", ["tcc", "cc"]) From 2953ec8198a67463e661b117f675ca5d0439c22a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 20:56:24 +0200 Subject: [PATCH 16/30] Verify numeric-then-truth portable Boolean arithmetic --- plans/portable-DSLs-first-version.md | 13 +++++++++++ tests/test_dsl_portable.py | 5 ++++ tests/test_portable_artifact.py | 35 ++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 609bdda7d..6a4e7d5da 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -260,6 +260,19 @@ failures or new native build warnings. `audit/bool_output_fraction` preserves an unresolved Boolean-output arithmetic discrepancy, not a normative rule; that domain and broader mixed-type promotion remain publication gates. +The next slice implements the user-selected numeric-then-truth Boolean contract. +Boolean operands are numeric zero/one, fractional literals and intermediates +stay numeric, and truth conversion occurs only at a Boolean output or explicit +`bool()` call. Boolean-only/integral-literal contexts compute in int64; +fractional-literal contexts compute in float64. Locals and cast arguments retain +numeric values, and a `2**53` cancellation regression checks exact integral +intermediates. `bool_output_fraction` is promoted from the audit corpus, with +four additional shared fixtures and scalar/vector backend matrices. Cache +version is 20. Validation passed 413 regular native tests, 322 AddressSanitizer +tests, and 2161 focused Python tests without expected failures or new native +compiler warnings. This closes the sampled Boolean-output discrepancy, not the +remaining arbitrary mixed-type promotion and overflow contracts. + ## 1. Goal and scope Establish a small, versioned miniexpr kernel language that executes independently diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 2c1e92795..088335025 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -61,6 +61,11 @@ "bool_float_product", "bool_float_difference", "bool_float_local_neg", + "bool_output_fraction", + "bool_output_add", + "bool_output_cast", + "bool_numeric_local", + "bool_numeric_int_cast", "while_cap_exact", "while_cap_exceeded", "while_cap_continue", diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index e5d757cc7..d7d2230c5 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -351,6 +351,41 @@ def test_optional_adapter_availability(): blosc2.PortableKernel.from_json("{}", jit=False) +@pytest.mark.parametrize("compiler", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize("count", [1, 2, 257]) +@pytest.mark.parametrize( + ("body", "samples"), + [ + (" return x + 0.5\n", [True, True]), + (" return x * 0.5\n", [False, True]), + (" return bool(x + 0.5)\n", [True, True]), + (" return (x + x) == 2\n", [False, True]), + (" return (x * 0.5) == 0.5\n", [False, True]), + (" return int(x + 0.5) == 0\n", [True, False]), + (" half = x * 0.5\n return half == 0.5\n", [False, True]), + (" half = x * 0.5\n return half * 2 == x\n", [True, True]), + (" return x - 1\n", [True, False]), + (" return -x\n", [False, True]), + (" return (x + 9007199254740992) - 9007199254740992\n", [False, True]), + ], +) +def test_bool_output_numeric_arithmetic(native_artifacts, compiler, jit, count, body, samples): + artifact = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n{body}").export( + {"x": "bool"}, "bool" + ) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + if jit: + assert kernel.has_jit + values = np.resize(np.array([False, True]), count) + expected = np.resize(np.array(samples), count) + for _ in range(2): + actual = kernel.evaluate({"x": values}) + np.testing.assert_array_equal(actual, expected) + assert actual.dtype == np.dtype("bool") + assert np.all(actual.view("uint8") <= 1) + + @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) @pytest.mark.parametrize("input_dtype", ["bool", "int32", "int64", "float32", "float64"]) From d2c2534fa9370a45c117dff3a34a512da2ea9a72 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 21:18:33 +0200 Subject: [PATCH 17/30] Integrate the frozen portable DSL 0.1 profile and native loader --- CMakeLists.txt | 6 +- doc/reference/index.rst | 1 + doc/reference/portable_dsl.rst | 85 ++++++++++++++++++++ plans/portable-DSLs-first-version.md | 32 +++++++- src/blosc2/blosc2_ext.pyx | 2 +- src/blosc2/dsl_kernel.py | 4 +- src/blosc2/portable_kernel.py | 6 +- tests/test_dsl_portable.py | 12 ++- tests/test_portable_artifact.py | 113 +++++++++++++++------------ 9 files changed, 201 insertions(+), 60 deletions(-) create mode 100644 doc/reference/portable_dsl.rst diff --git a/CMakeLists.txt b/CMakeLists.txt index fc66a804a..07b6e0aae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,6 +118,7 @@ set(MINIEXPR_BUILD_SHARED OFF CACHE BOOL "Build miniexpr shared library" FORCE) set(MINIEXPR_BUILD_TESTS OFF CACHE BOOL "Build miniexpr tests" FORCE) set(MINIEXPR_BUILD_EXAMPLES OFF CACHE BOOL "Build miniexpr examples" FORCE) set(MINIEXPR_BUILD_BENCH OFF CACHE BOOL "Build miniexpr benchmarks" FORCE) +option(MINIEXPR_BUILD_ARTIFACT "Build portable DSL artifact support" ON) # Keep miniexpr's and C-Blosc2's bundled files inside the blosc2 package. # Without this, their install rules use the default CMAKE_INSTALL_LIBDIR # ("lib"), which scikit-build places at site-packages/lib. @@ -141,7 +142,7 @@ endif() FetchContent_Declare(miniexpr GIT_REPOSITORY https://github.com/Blosc/miniexpr.git - GIT_TAG 3f4db93a628aedc662b7d27a5f35c97ffaeb0424 + GIT_TAG 7543a9592410bdfd72f7a7319343dfece55b09e4 # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr ) FetchContent_MakeAvailable(miniexpr) @@ -151,6 +152,9 @@ target_link_libraries(blosc2_ext PRIVATE miniexpr_static) if(TARGET miniexpr_artifact) target_link_libraries(blosc2_ext PRIVATE miniexpr_artifact) target_compile_definitions(blosc2_ext PRIVATE B2_HAVE_PORTABLE_ARTIFACT=1) + FetchContent_GetProperties(yyjson SOURCE_DIR B2_YYJSON_SOURCE_DIR) + install(FILES "${B2_YYJSON_SOURCE_DIR}/LICENSE" + DESTINATION ${SKBUILD_METADATA_DIR}/licenses/miniexpr RENAME LICENSE-YYJSON) endif() if(APPLE) target_link_libraries(blosc2_ext PRIVATE "-framework Accelerate") diff --git a/doc/reference/index.rst b/doc/reference/index.rst index a0ba44c75..f7d3e74d4 100644 --- a/doc/reference/index.rst +++ b/doc/reference/index.rst @@ -12,4 +12,5 @@ API Reference save_load utilities dsl_syntax + portable_dsl jit diff --git a/doc/reference/portable_dsl.rst b/doc/reference/portable_dsl.rst new file mode 100644 index 000000000..1a95d5560 --- /dev/null +++ b/doc/reference/portable_dsl.rst @@ -0,0 +1,85 @@ +Portable DSL artifacts +====================== + +Portable profile 0.1 is a conservative, versioned subset of the full native DSL. +Artifacts are standalone UTF-8 JSON, executable through native miniexpr in C or +Python, without reconstructing or executing Python functions on import. + +.. code-block:: python + + import blosc2 + import numpy as np + + author = blosc2.DSLKernel.from_source("def affine(x):\n return x * 2.0 - 1.0\n") + artifact = author.export({"x": "float64"}, "float64") + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + values = kernel.evaluate({"x": np.arange(4, dtype="float64")}) + # array([-1., 1., 3., 5.]) + +The result is an eager NumPy array. Bindings are named and require exact dtypes +and matching shapes. Supported dtypes are ``bool``, ``int32``, ``int64``, +``float32``, and ``float64``; all parameters and captures share one dtype, while +output may differ. Captured supported Python/NumPy scalars are snapshotted into +typed constants, never live callbacks or arrays. Integers use range-checked +decimal strings; floats use exact IEEE-754 hexadecimal bits. + +Release boundary +---------------- + +Supported features include elementwise arithmetic, comparisons and chains, +Boolean logic, local assignments, branches, bounded loops, and explicit output +conversions. Boolean operands participate in numeric arithmetic as zero/one; +truth conversion occurs at a Boolean output or explicit ``bool()`` call. + +Excluded features include transcendental calls (including ``sin``/``cos``), +nested ``int()``/``float()`` casts, expression arguments to casts in floating-input +signatures, mixed-input dtypes, uncertified mixed computation intermediates, +integer division, float32 arithmetic inside predicates, reductions, indexing, +floating-spelled literals in integral-input signatures, +shape symbols, and external functions. A separately assigned float32 arithmetic +local can be compared after rounding. Excluded source is rejected both during +export and independently by native import; there is no Python fallback. + +Integer overflow, out-of-range narrowing, non-finite/out-of-range float-to-integer +casts, zero divisors, and non-default floating rounding modes are outside the +runtime contract. Callers must keep data and intermediates in the defined domains; +static validation cannot prove those conditions. This is not a sandbox for +untrusted source or arbitrary data. NaN payloads are not guaranteed. + +Use ``blosc2.validate_portable_dsl(source, input_dtypes, output_dtype)`` to check +membership without execution or JIT. It returns validity, a status category, +and optional source diagnostics. Import/export errors raise +``blosc2.PortableArtifactError`` with a stable ``status`` attribute. Missing +returns, zero range steps, and loop-cap failures are runtime evaluation errors; +failed output contents are unspecified. Empty evaluations validate bindings but +do not execute the kernel. + +Builds and execution +-------------------- + +The native artifact loader is enabled by default in Python package builds. +Custom builds may disable it with ``MINIEXPR_BUILD_ARTIFACT=OFF``; import then +raises ``NotImplementedError``. The optional adapter uses pinned, MIT-licensed +yyjson; the raw native compiler does not depend on it. + +The interpreter is the baseline. TCC/CC are optional accelerators, selected with +``jit`` and native compiler pragmas. Missing accelerators can fall back to native +interpretation. Semantic evaluation errors never retry the kernel. Artifacts +enforce strict FP independently of host defaults. Host while-loop limits remain +execution policy rather than artifact constants. + +The authoritative language and artifact specifications are maintained in the +pinned miniexpr dependency under ``doc/dsl-spec/0.1.md`` and +``doc/dsl-spec/artifact-0.1.md``. The feature boundary is frozen; certification +on additional release platforms is tracked by CI. Broader numeric features and +lazy/container integration are deferred to a later profile. + +API +--- + +.. autoclass:: blosc2.PortableKernel + :members: + +.. autoclass:: blosc2.PortableArtifactError + +.. autofunction:: blosc2.validate_portable_dsl diff --git a/plans/portable-DSLs-first-version.md b/plans/portable-DSLs-first-version.md index 6a4e7d5da..d2deb0e8c 100644 --- a/plans/portable-DSLs-first-version.md +++ b/plans/portable-DSLs-first-version.md @@ -1,8 +1,34 @@ # Portable DSL v0.1: first implementation plan -**Status:** Implementation plan for the `portable-dsls` branch. Proposed API names -and wire-format details below remain experimental until the first implementation -and conformance tests establish them. +**Status: conservative 0.1 boundary frozen; release integration in progress.** +The normative specifications are `../miniexpr/doc/dsl-spec/0.1.md` and +`artifact-0.1.md`; the development history below is not the current release scope. + +### Closed scope and release checklist + +- Native source and compiled-tree gates exclude transcendental calls, nested + numeric casts, floating expression arguments to truth casts, mixed computation + intermediates, and unlowered float32/division predicate arithmetic. +- Overflow, out-of-domain casts, zero divisors, and non-default rounding are + explicit runtime-domain exclusions, not promises of wraparound or trapping. +- `frozen-excluded.txt` separates preserved full-language regressions from + conformance. Negative tests verify validation, export, and native import; + unsupported artifacts cannot enter via a bypassed authoring frontend. +- Python package builds enable the optional native artifact adapter by default, + retaining an explicit OFF build option and including yyjson's MIT license. +- Native frozen revision published: `7543a9592410bdfd72f7a7319343dfece55b09e4`. + Python pins that immutable revision, including platform test-harness fixes. +- Stock wheel verification passed without a local native source override: + artifact support defaults ON, yyjson's license is included in wheel metadata, + and interpreter/JIT execution works after moving the build tree. The isolated + wheel passed 2171 focused Python tests; local native and sanitizer suites passed + 431 and 340 tests respectively. Documentation builds with pre-existing warnings. +- Host CI now enables native artifacts on Linux/macOS/Windows and ARM64 jobs. + Local certification is macOS arm64; other platform jobs must be green before + announcing cross-platform release certification. +- Windows conformance certifies the interpreter baseline, not TCC acceleration. +- Numeric scope is closed. Broader math/casts, overflow policy, arbitrary mixed + promotion, and lazy/container integration are later-profile work, not 0.1 gates. ### Implementation progress diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index 7b74a63aa..d7cfbeae1 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -989,7 +989,7 @@ cdef extern from *: def validate_portable_dsl_source(source, input_dtypes, output_dtype, version): - """Native-only draft profile validation; no arrays, JIT, or Python execution.""" + """Native-only portable 0.1 validation; no arrays, JIT, or Python execution.""" cdef Py_ssize_t n = len(input_dtypes) if n > 128: raise ValueError("Too many portable DSL inputs (maximum 128)") diff --git a/src/blosc2/dsl_kernel.py b/src/blosc2/dsl_kernel.py index 6a4efa76d..153e70f45 100644 --- a/src/blosc2/dsl_kernel.py +++ b/src/blosc2/dsl_kernel.py @@ -943,7 +943,7 @@ def export(self, input_dtypes, output_dtype, *, capture_dtypes=None, constants=N collision-free parameters. ``capture_dtypes`` overrides scalar type inference (Python bool/int/float infer bool/int64/float64 respectively; supported NumPy scalars retain their dtype). Arrays, objects, and external - callbacks cannot be captures. The draft requires homogeneous parameter + callbacks cannot be captures. Profile 0.1 requires homogeneous parameter dtypes. Import via :meth:`blosc2.PortableKernel.from_json`. """ from .portable_kernel import export_portable_kernel @@ -1050,7 +1050,7 @@ def validate_dsl(func): def validate_portable_dsl(source, input_dtypes, output_dtype, *, language_version="0.1"): - """Check raw native source and a typed signature against the draft portable profile. + """Check raw native source and a typed signature against conservative portable profile 0.1. Return a dictionary with ``valid``, ``status``, ``line``, ``column``, and ``error``. Validation neither executes a kernel nor invokes a JIT compiler. diff --git a/src/blosc2/portable_kernel.py b/src/blosc2/portable_kernel.py index f50506b20..bdc739a56 100644 --- a/src/blosc2/portable_kernel.py +++ b/src/blosc2/portable_kernel.py @@ -2,7 +2,7 @@ # Copyright (c) 2026, Blosc Development Team # SPDX-License-Identifier: BSD-3-Clause ####################################################################### -"""Experimental portable DSL artifact authoring and native-only execution.""" +"""Conservative portable DSL 0.1 authoring and native-only execution.""" from __future__ import annotations @@ -268,8 +268,8 @@ class PortableKernel: """An owned, typed native artifact; no originating function or Python fallback. Use ``from_json`` to import and ``evaluate`` to return NumPy values. Inputs - are same-shaped logical arrays, bound by name. The experimental profile and - native adapter must be available in the installed extension. + are same-shaped logical arrays, bound by name. The conservative 0.1 profile + is enforced by the native adapter (enabled by default in package builds). """ @classmethod diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 088335025..1dcb2de01 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -85,6 +85,11 @@ "missing_return_hybrid_ok", ] TYPES = ("bool", "int32", "int64", "float32", "float64") +EXCLUDED = ( + set((CORPUS / "frozen-excluded.txt").read_text().split()) + if (CORPUS / "frozen-excluded.txt").is_file() + else set() +) CASES += [f"convert_{input_type}_{output_type}" for input_type in TYPES for output_type in TYPES] @@ -144,6 +149,7 @@ def test_native_source_corpus(case, jit, monkeypatch): str(corpus_source(case)), str(CORPUS / f"{case}.txt"), "on" if jit else "off", + *(["native"] if case in EXCLUDED else []), ], check=True, capture_output=True, @@ -220,7 +226,9 @@ def test_portable_profile_corpus(portable_validator, case): nvars = int(words[4]) inputs = dict.fromkeys(words[5 : 5 + nvars], input_type) info = portable_validator(corpus_source(case).read_text(), inputs, output_type) - assert info["valid"] == (outcome != "compile_error"), info + assert info["valid"] == (outcome != "compile_error" and case not in EXCLUDED), info + if case in EXCLUDED: + assert info["status"] == "unsupported_feature" @pytest.mark.parametrize( @@ -233,7 +241,7 @@ def test_portable_profile_corpus(portable_validator, case): ("return _flat_idx + x", "unsupported_feature"), ("return callback(x)", "unsupported_feature"), ("return CAPTURE + x", "invalid_source"), - ("return sin()", "invalid_source"), + ("return sin()", "unsupported_feature"), ], ) def test_portable_profile_rejections(portable_validator, body, status): diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index d7d2230c5..411d17f55 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -20,10 +20,54 @@ ) ) ARTIFACTS = CORPUS.parent / "portable-artifacts" +EXCLUDED = ( + set((CORPUS / "frozen-excluded.txt").read_text().split()) + if (CORPUS / "frozen-excluded.txt").is_file() + else set() +) SCALE = 2.0 BIAS = -1.0 +def assert_frozen_rejection(source, inputs, output): + """Check both authoring and native import, without disguising setup errors.""" + info = blosc2.validate_portable_dsl(source, inputs, output) + assert not info["valid"], info + assert info["status"] == "unsupported_feature", info + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.DSLKernel.from_source(source).export(inputs, output) + assert error.value.status == "unsupported_feature" + # Use a valid envelope, then replace only its source. Import must independently + # enforce the native frozen boundary even when authoring is bypassed. + names = ", ".join(inputs) + identity = f"def k({names}):\n return {next(iter(inputs))}\n" + envelope = json.loads(blosc2.DSLKernel.from_source(identity).export(inputs, output)) + envelope["source"] = source + envelope["entry_point"] = ast.parse(source).body[0].name + with pytest.raises(blosc2.PortableArtifactError) as error: + blosc2.PortableKernel.from_json(json.dumps(envelope), jit=False) + assert error.value.status == "unsupported_requirement" + + +@pytest.mark.parametrize( + ("expression", "input_dtype", "output_dtype"), + [ + ("sin(x)", "float64", "float64"), + ("cos(x)", "float32", "float32"), + ("int(x + 0.25)", "float64", "int64"), + ("float(float(x))", "float32", "float32"), + ("bool((x + 1.0) - x)", "float32", "bool"), + ("(x + 1.0) > x", "float32", "bool"), + ("(x / 2) > 0", "float64", "bool"), + ("x + 0.5", "int64", "int64"), + ("x == 9007199254740993.0", "int64", "bool"), + ("float(x)", "int64", "float32"), + ], +) +def test_frozen_boundary_without_corpus(native_artifacts, expression, input_dtype, output_dtype): + assert_frozen_rejection(f"def k(x):\n return {expression}\n", {"x": input_dtype}, output_dtype) + + @blosc2.dsl_kernel def affine(x): return x * SCALE + BIAS @@ -66,12 +110,9 @@ def test_capture_closure_and_frontend_sugar(native_artifacts): def trig(x): return np.sin(x) * scale - artifact = trig.export({"x": "float32"}, "float32") - scale = np.float32(99.0) - kernel = blosc2.PortableKernel.from_json(artifact, jit=False) - x = np.array([0.0, 0.25, 1.0], dtype=np.float32) - np.testing.assert_allclose(kernel.evaluate({"x": x}), np.sin(x) * 2, rtol=2e-6) - assert "np.sin" not in kernel.source + with pytest.raises(blosc2.PortableArtifactError) as error: + trig.export({"x": "float32"}, "float32") + assert error.value.status == "unsupported_feature" def test_capture_collision_and_comments(native_artifacts): @@ -371,6 +412,9 @@ def test_optional_adapter_availability(): ], ) def test_bool_output_numeric_arithmetic(native_artifacts, compiler, jit, count, body, samples): + if "int(x +" in body: + assert_frozen_rejection(f"# me:compiler={compiler}\ndef k(x):\n{body}", {"x": "bool"}, "bool") + return artifact = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n{body}").export( {"x": "bool"}, "bool" ) @@ -445,6 +489,9 @@ def test_float_cast_arithmetic_rounding(native_artifacts, compiler, jit, count, "negation": "float(-(x * 0.1))", } source = f"# me:compiler={compiler}\ndef k(x):\n return {expressions[operation]}\n" + if operation != "leaf": + assert_frozen_rejection(source, {"x": "float32"}, output_dtype) + return artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) if jit: @@ -482,22 +529,13 @@ def test_float_cast_arithmetic_rounding(native_artifacts, compiler, jit, count, @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @pytest.mark.parametrize("jit", [False, True]) @pytest.mark.parametrize("count", [1, 2, 257]) -def test_float32_leaf_math(native_artifacts, case, compiler, jit, count): - """Leaf math rounds in float32 before widening or comparing.""" +def test_math_is_outside_frozen_profile(native_artifacts, case, compiler, jit, count): if not CORPUS.is_dir(): pytest.skip("Native fixtures unavailable") words = (CORPUS / f"{case}.txt").read_text().split() input_dtype, output_dtype = words[1:3] - rows = np.array(words[6:]).reshape(int(words[3]), 2) source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") - artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) - kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit - values = np.resize(np.array(rows[:, 0], dtype=input_dtype), count) - expected = np.resize(np.array(rows[:, 1], dtype=output_dtype), count) - for _ in range(2): - np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) @pytest.mark.parametrize( @@ -540,6 +578,9 @@ def test_typed_arithmetic_artifact(native_artifacts, case, compiler, jit): input_dtype, output_dtype = words[1:3] rows = np.array(words[6:]).reshape(int(words[3]), 2) source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + if case in EXCLUDED: + assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) + return artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) assert kernel.has_jit == jit @@ -640,13 +681,7 @@ def test_while_cap_hybrid_cleanup(native_artifacts, monkeypatch, compiler): f"# me:compiler={compiler}\ndef k(x):\n value = sin(x)\n n = 0\n" " while n < x:\n n = n + 1\n return value\n" ) - artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, "float64") - kernel = blosc2.PortableKernel.from_json(artifact, jit=True) - assert kernel.has_jit - for _ in range(4): - with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): - kernel.evaluate({"x": np.full(257, 4.0)}) - np.testing.assert_array_equal(kernel.evaluate({"x": np.zeros(257)}), np.zeros(257)) + assert_frozen_rejection(source, {"x": "float64"}, "float64") @pytest.mark.parametrize("compiler", ["tcc", "cc"]) @@ -736,33 +771,18 @@ def test_while_cap_control_flow(native_artifacts, monkeypatch, compiler, jit, bo ("if (int(x) / 2) > 0:\n return 1.0\n return 0.0", "float64", [0, 0, 0, 1, 1]), ], ) -def test_unsupported_division_lowering_uses_interpreter( - native_artifacts, compiler, body, output_dtype, expected -): +def test_unsupported_division_is_rejected(native_artifacts, compiler, body, output_dtype, expected): source = f"# me:compiler={compiler}\ndef k(x):\n {body}\n" - artifact = blosc2.DSLKernel.from_source(source).export({"x": "float64"}, output_dtype) - kernel = blosc2.PortableKernel.from_json(artifact, jit=True) - assert not kernel.has_jit - np.testing.assert_array_equal( - kernel.evaluate({"x": np.array([-3.75, -1.75, 0, 1.75, 3.75])}), - np.array(expected, dtype=output_dtype), - ) + assert_frozen_rejection(source, {"x": "float64"}, output_dtype) @pytest.mark.parametrize("input_dtype", ["float32", "float64"]) @pytest.mark.parametrize("output_dtype", ["float32", "float64"]) @pytest.mark.parametrize("count", [1, 5, 257]) @pytest.mark.parametrize("jit", [False, True]) -def test_nested_conversion_buffer_width(native_artifacts, input_dtype, output_dtype, count, jit): +def test_nested_conversion_is_rejected(native_artifacts, input_dtype, output_dtype, count, jit): source = "def k(x):\n return float(int(x) / 2)\n" - artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) - kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - # Nested casts remain intentionally outside the typed JIT arithmetic slice. - assert not kernel.has_jit - values = np.resize(np.array([-1.75, -0.25, 0.25, 1.75, 4.75], dtype=input_dtype), count) - expected = np.resize(np.array([-0.5, 0, 0, 0.5, 2], dtype=output_dtype), count) - for _ in range(2): - np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) @pytest.mark.parametrize( @@ -776,14 +796,11 @@ def test_nested_conversion_buffer_width(native_artifacts, input_dtype, output_dt @pytest.mark.parametrize("input_dtype", ["float32", "float64"]) @pytest.mark.parametrize("output_dtype", ["bool", "int32", "int64", "float32", "float64"]) @pytest.mark.parametrize("jit", [False, True]) -def test_value_cast_argument_semantics( +def test_floating_expression_casts_are_rejected( native_artifacts, expression, expected, input_dtype, output_dtype, jit ): source = f"# me:compiler=tcc\ndef k(x):\n return {expression}\n" - artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) - kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - values = np.array([-1.75, -0.25, 0.25, 1.75, 4.75], dtype=input_dtype) - np.testing.assert_array_equal(kernel.evaluate({"x": values}), np.array(expected, dtype=output_dtype)) + assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) @pytest.mark.parametrize("jit", [False, True]) From ac7756134abceb79082be6886e7ad5a3973c2309 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 21:38:46 +0200 Subject: [PATCH 18/30] Fix JIT reduction safety and streamline portable test coverage --- doc/reference/jit.rst | 12 +- src/blosc2/lazyexpr.py | 28 ++- tests/ndarray/test_jit_options.py | 42 +++++ tests/test_dsl_portable.py | 76 +------- tests/test_portable_artifact.py | 294 ++++++++++++------------------ 5 files changed, 195 insertions(+), 257 deletions(-) diff --git a/doc/reference/jit.rst b/doc/reference/jit.rst index ecae2de28..49eeb939f 100644 --- a/doc/reference/jit.rst +++ b/doc/reference/jit.rst @@ -145,6 +145,12 @@ accept them through ``**kwargs``. Eligible lazy-expression reductions, such as settings tune execution rather than array storage; they are not options to :func:`blosc2.empty`. +Scalar miniexpr reductions currently use the native interpreter, even when JIT +is requested. Elementwise DSL returns (including automatically lifted expressions) +write a block of values and cannot safely target a single reduction accumulator. +Other execution options still apply, and non-scalar reduction paths retain their +usual evaluation policies. + ``jit=None`` (default) Use the entry point's default policy. DSL kernels, including the real-valued ramps used by ``arange`` and ``linspace``, try JIT. Plain expressions do not @@ -268,9 +274,9 @@ controls have different scopes; neither is an indication that JIT actually ran. - Disable applicable process-local positive-cache reuse. This does not disable persistent CC disk-cache reuse. * - ``ME_DSL_JIT_COMPILER`` - - Advanced miniexpr-wide compiler override, ``tcc`` or ``cc``. It overrides - native compiler selection in DSL pragmas, including the selection emitted - by Python's ``jit_backend`` option. Prefer per-call options ordinarily. + - Advanced miniexpr-wide compiler default, ``tcc`` or ``cc``. It overrides + Python's ``jit_backend`` selection, but an explicit author-written + ``# me:compiler=tcc|cc`` pragma remains authoritative. Tracing and restricted environments ----------------------------------- diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py index 732c1141e..9f94f5761 100644 --- a/src/blosc2/lazyexpr.py +++ b/src/blosc2/lazyexpr.py @@ -1530,6 +1530,10 @@ def fill_chunk_operands( def _apply_jit_backend_pragma(expression: str, inputs: dict, jit_backend: str | None) -> str: + # Author-written pragmas are authoritative. Python's synthesized pragma is + # only a local default and must not override the native environment policy. + if "# me:compiler=" in expression: + return expression if jit_backend is None: return expression if jit_backend == "js": @@ -1538,11 +1542,15 @@ def _apply_jit_backend_pragma(expression: str, inputs: dict, jit_backend: str | if jit_backend not in ("tcc", "cc"): raise ValueError("jit_backend must be one of: None, 'tcc', 'cc', 'js'") + env_backend = os.environ.get("ME_DSL_JIT_COMPILER", "").strip().lower() + if env_backend in ("tcc", "cc"): + jit_backend = env_backend + pragma = f"# me:compiler={jit_backend}\n" - stripped = expression.lstrip() + stripped = "\n".join( + line for line in expression.splitlines() if not line.lstrip().startswith("#") + ).lstrip() if stripped.startswith("def "): - if "# me:compiler=" in expression: - return expression return pragma + expression params = ", ".join(k for k, v in inputs.items() if hasattr(v, "dtype")) return f"{pragma}def __me_auto({params}):\n return {expression}" @@ -2733,8 +2741,9 @@ def reduce_slices( # noqa: C901 if ne_args is None: ne_args = {} fp_accuracy = kwargs.pop("fp_accuracy", blosc2.FPAccuracy.DEFAULT) - jit = kwargs.pop("jit", None) - jit_backend = kwargs.pop("jit_backend", None) + # Scalar native reductions cannot use elementwise DSL JIT output buffers. + kwargs.pop("jit", None) + kwargs.pop("jit_backend", None) where: dict | None = kwargs.pop("_where_args", None) reduce_op = reduce_args.pop("op") reduce_op_str = reduce_args.pop("op_str", None) @@ -2912,8 +2921,13 @@ def reduce_slices( # noqa: C901 expression_miniexpr = f"{reduce_op_str}(where({expression}, _where_x, _where_y))" else: expression_miniexpr = f"{reduce_op_str}({expression})" - expression_miniexpr = _apply_jit_backend_pragma(expression_miniexpr, operands, jit_backend) - res_eval._set_pref_expr(expression_miniexpr, operands, fp_accuracy, aux_reduc, jit=jit) + # Keep reductions as native scalar expressions. A DSL return + # broadcasts a reduced value across the output block, whereas this + # prefilter supplies only one accumulator slot per block. Explicit + # JIT_ON also auto-lifts plain expressions in the native compiler, + # so use the scalar interpreter until reduction JIT has a separate + # output-cardinality contract. + res_eval._set_pref_expr(expression_miniexpr, operands, fp_accuracy, aux_reduc, jit=False) prefilter_set = True # print("expr->miniexpr:", expression, reduce_op, fp_accuracy) # Data won't even try to be compressed, so buffers can be unitialized and reused diff --git a/tests/ndarray/test_jit_options.py b/tests/ndarray/test_jit_options.py index dda096266..57a011376 100644 --- a/tests/ndarray/test_jit_options.py +++ b/tests/ndarray/test_jit_options.py @@ -353,6 +353,48 @@ def test_environment_overrides_and_tcc_ignores_cc_settings(tmp_path, capfd, monk assert capfd.readouterr() == ("", "") +@pytest.mark.parametrize("backend", ["tcc", "cc"]) +def test_synthesized_backend_respects_environment_and_source(monkeypatch, backend): + from blosc2.lazyexpr import _apply_jit_backend_pragma + + monkeypatch.setenv("ME_DSL_JIT_COMPILER", backend) + other = "cc" if backend == "tcc" else "tcc" + inputs = {"x": np.arange(4, dtype=np.float64)} + assert _apply_jit_backend_pragma("x + 1", inputs, other).startswith(f"# me:compiler={backend}\n") + source = f"# me:compiler={other}\ndef k(x):\n return x + 1\n" + assert _apply_jit_backend_pragma(source, inputs, backend) == source + commented = "# me:fp=strict\ndef k(x):\n return x\n" + assert _apply_jit_backend_pragma(commented, inputs, other) == f"# me:compiler={backend}\n{commented}" + + +@pytest.mark.parametrize("backend", ["tcc", "cc"]) +@pytest.mark.parametrize("jit", [False, True]) +def test_reduction_backend_preserves_scalar_accumulator(monkeypatch, backend, jit): + import importlib + + lazyexpr = importlib.import_module("blosc2.lazyexpr") + + def reject_elementwise_wrapper(*args): + pytest.fail("Scalar reductions must not be wrapped in elementwise DSL source") + + monkeypatch.setattr(lazyexpr, "_apply_jit_backend_pragma", reject_elementwise_wrapper) + original = blosc2.NDArray._set_pref_expr + calls = [] + + def capture(self, expression, inputs, fp_accuracy, aux_reduc=None, **kwargs): + assert aux_reduc is not None + assert kwargs["jit"] is False # Prevent native auto-lifting as well. + calls.append(expression) + return original(self, expression, inputs, fp_accuracy, aux_reduc, **kwargs) + + monkeypatch.setattr(blosc2.NDArray, "_set_pref_expr", capture) + values = np.arange(128, dtype=np.float64) + expr = blosc2.asarray(values) * 2 + 1 + assert expr.sum(jit=jit, jit_backend=backend) == (values * 2 + 1).sum() + assert calls + assert all(expression.startswith("sum(") for expression in calls) + + @pytest.mark.skipif(blosc2.IS_WASM or os.name == "nt", reason="POSIX compiler-output injection") def test_compiler_output(tmp_path, capfd): compiler = tmp_path / "bad cc's executable" diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 1dcb2de01..8052b20a5 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -1,4 +1,4 @@ -"""Initial raw-source portable conformance; the corpus is owned by miniexpr.""" +"""Representative Python host conformance; exhaustive fixtures belong to miniexpr.""" import os import subprocess @@ -18,71 +18,25 @@ ) +# Native CTest runs every fixture under its supported execution engines. Python +# tests only distinct host boundaries here, plus every exact output conversion; +# numerical/JIT scenarios are covered separately by test_portable_artifact.py. CASES = [ "affine", "chains", "loops", - "int32", - "int64", - "float32", - "bool", - "compare", "special_floats", "flow", - "boolean_results", - "cast_int", "math", - "division_int", "division_bool", "division_literals", "division_locals", - "division_loop", - "division_float32", - "division_condition", - "cast_argument_int", - "cast_argument_bool", - "cast_argument_nested", - "cast_integer_exact", - "cast_nonfinite_truth", - "math_widen", - "math_condition", - "math_local_widen", - "arithmetic_round", - "arithmetic_decimal", - "arithmetic_scalar_sub", - "arithmetic_local_round", - "arithmetic_widen", - "arithmetic_float_cast", - "arithmetic_float_nested", - "arithmetic_float_local", - "arithmetic_float_widen", - "bool_float_sum", - "bool_float_neg", - "bool_float_product", - "bool_float_difference", - "bool_float_local_neg", - "bool_output_fraction", - "bool_output_add", - "bool_output_cast", - "bool_numeric_local", - "bool_numeric_int_cast", - "while_cap_exact", "while_cap_exceeded", - "while_cap_continue", - "while_cap_hybrid", - "while_cap_chain", - "masked_local_chain", - "masked_bool_chain", "unknown_name", - "invalid_index", "zero_step", - "missing_return", - "missing_return_ok", "missing_return_mixed", "missing_return_loop", "missing_return_loop_ok", - "missing_return_hybrid", - "missing_return_hybrid_ok", ] TYPES = ("bool", "int32", "int64", "float32", "float64") EXCLUDED = ( @@ -115,8 +69,8 @@ def assert_typed_result(actual, expected, *, exact=False): @pytest.mark.parametrize("case", CASES) -@pytest.mark.parametrize("jit", [False, True]) -def test_native_source_corpus(case, jit, monkeypatch): +def test_native_source_corpus(case, portable_validator, monkeypatch): + jit = True if case.startswith("while_cap_"): monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") if not CORPUS.is_dir(): @@ -126,6 +80,10 @@ def test_native_source_corpus(case, jit, monkeypatch): outcome, input_type, output_type = words[:3] count, nvars = map(int, words[3:5]) names = words[5 : 5 + nvars] + info = portable_validator(source, dict.fromkeys(names, input_type), output_type) + assert info["valid"] == (outcome != "compile_error" and case not in EXCLUDED), info + if case in EXCLUDED: + assert info["status"] == "unsupported_feature" rows = np.array(words[5 + nvars :]).reshape(count, nvars + 1) input_dtype, output_dtype = np.dtype(input_type), np.dtype(output_type) expected = typed_values(rows[:, -1], output_dtype) @@ -217,20 +175,6 @@ def portable_validator(): return blosc2.validate_portable_dsl -@pytest.mark.parametrize("case", CASES) -def test_portable_profile_corpus(portable_validator, case): - if not CORPUS.is_dir(): - pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") - words = (CORPUS / f"{case}.txt").read_text().split() - outcome, input_type, output_type = words[:3] - nvars = int(words[4]) - inputs = dict.fromkeys(words[5 : 5 + nvars], input_type) - info = portable_validator(corpus_source(case).read_text(), inputs, output_type) - assert info["valid"] == (outcome != "compile_error" and case not in EXCLUDED), info - if case in EXCLUDED: - assert info["status"] == "unsupported_feature" - - @pytest.mark.parametrize( ("body", "status"), [ diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 411d17f55..260dd88f5 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -2,6 +2,13 @@ # Copyright (c) 2026, Blosc Development Team # SPDX-License-Identifier: BSD-3-Clause ####################################################################### +"""Portable host integration, with separate semantic and execution matrices. + +The exhaustive shared DSL corpus is certified by native miniexpr CTest. Here, +boundary values live in compact dtype/operator scenarios; engine/length checks +are independent. Rejected features do not multiply across unused JIT/size axes. +""" + import ast import json import os @@ -20,11 +27,6 @@ ) ) ARTIFACTS = CORPUS.parent / "portable-artifacts" -EXCLUDED = ( - set((CORPUS / "frozen-excluded.txt").read_text().split()) - if (CORPUS / "frozen-excluded.txt").is_file() - else set() -) SCALE = 2.0 BIAS = -1.0 @@ -62,6 +64,10 @@ def assert_frozen_rejection(source, inputs, output): ("x + 0.5", "int64", "int64"), ("x == 9007199254740993.0", "int64", "bool"), ("float(x)", "int64", "float32"), + ("float(-(x * 0.1))", "float32", "float32"), + ("bool(int(x + 0.25))", "float64", "bool"), + ("float(int(x) / 2)", "float32", "float32"), + ("int(x) / 2", "float64", "int64"), ], ) def test_frozen_boundary_without_corpus(native_artifacts, expression, input_dtype, output_dtype): @@ -392,9 +398,45 @@ def test_optional_adapter_availability(): blosc2.PortableKernel.from_json("{}", jit=False) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) +# Execution engines and sizes are independent of the numeric scenario matrix. +# Exercise scalar, short-vector, and tile-tail execution, including repeated +# handles, Boolean bytes, float32 rounding, and local negative zero. +@pytest.mark.parametrize( + ("compiler", "jit"), + [ + pytest.param("tcc", False, id="interpreter"), + pytest.param("tcc", True, id="tcc"), + pytest.param("cc", True, id="cc"), + ], +) @pytest.mark.parametrize("count", [1, 2, 257]) +def test_numeric_execution_shapes(native_artifacts, compiler, jit, count): + cases = [ + ("bool", "bool", " half = x * 0.5\n return half\n", [True, False], [True, False]), + ( + "float32", + "float32", + " rounded = float(x) + 0.1\n return rounded - float(x)\n", + [2**24, -(2**24)], + [0.0, 0.0], + ), + ("float64", "float64", " truth = bool(x)\n return -truth\n", [0.0, 1.0], [-0.0, -1.0]), + ] + for input_dtype, output_dtype, body, samples, expected in cases: + source = f"# me:compiler={compiler}\ndef k(x):\n{body}" + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) + assert kernel.has_jit == jit + for _ in range(2): + actual = kernel.evaluate({"x": np.resize(np.array(samples, dtype=input_dtype), count)}) + desired = np.resize(np.array(expected, dtype=output_dtype), count) + np.testing.assert_array_equal(actual, desired) + if output_dtype == "bool": + assert np.all(actual.view("uint8") <= 1) + else: + np.testing.assert_array_equal(np.signbit(actual), np.signbit(desired)) + + @pytest.mark.parametrize( ("body", "samples"), [ @@ -411,7 +453,8 @@ def test_optional_adapter_availability(): (" return (x + 9007199254740992) - 9007199254740992\n", [False, True]), ], ) -def test_bool_output_numeric_arithmetic(native_artifacts, compiler, jit, count, body, samples): +def test_bool_output_numeric_arithmetic(native_artifacts, body, samples): + compiler, jit, count = "tcc", False, 257 if "int(x +" in body: assert_frozen_rejection(f"# me:compiler={compiler}\ndef k(x):\n{body}", {"x": "bool"}, "bool") return @@ -430,16 +473,29 @@ def test_bool_output_numeric_arithmetic(native_artifacts, compiler, jit, count, assert np.all(actual.view("uint8") <= 1) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -@pytest.mark.parametrize("input_dtype", ["bool", "int32", "int64", "float32", "float64"]) -@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) -@pytest.mark.parametrize("count", [1, 10, 257]) -@pytest.mark.parametrize("operation", ["sum", "difference", "product", "negation"]) -@pytest.mark.parametrize("local", [False, True]) +# Test each dtype/operation once, rotating the independent execution paths and +# output widths. Backend/length coverage is tested separately below, not crossed +# with every numeric scenario. Locals and direct expressions both remain covered. +@pytest.mark.parametrize( + ("input_dtype", "operation", "compiler", "jit", "output_dtype", "local"), + [ + pytest.param( + dtype, + operation, + "cc" if (d + o) % 3 == 2 else "tcc", + (d + o) % 3 != 0, + "float32" if (d + o) % 2 == 0 else "float64", + (d // 2 + o) % 2 == 1, + id=f"{dtype}-{operation}", + ) + for d, dtype in enumerate(["bool", "int32", "int64", "float32", "float64"]) + for o, operation in enumerate(["sum", "difference", "product", "negation"]) + ], +) def test_bool_cast_floating_arithmetic( - native_artifacts, compiler, jit, input_dtype, output_dtype, count, operation, local + native_artifacts, compiler, jit, input_dtype, output_dtype, operation, local ): + count = 257 expressions = { "sum": "bool(x) + bool(x)", "difference": "bool(x) - bool(x)", @@ -476,22 +532,10 @@ def test_bool_cast_floating_arithmetic( np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -@pytest.mark.parametrize("count", [1, 10, 257]) @pytest.mark.parametrize("output_dtype", ["float32", "float64"]) -@pytest.mark.parametrize("operation", ["leaf", "nested", "repeated", "negation"]) -def test_float_cast_arithmetic_rounding(native_artifacts, compiler, jit, count, output_dtype, operation): - expressions = { - "leaf": "(float(x) + 0.1) - float(x)", - "nested": "float(x + 0.1) - float(x)", - "repeated": "float(float(x) + 0.1) - float(float(x))", - "negation": "float(-(x * 0.1))", - } - source = f"# me:compiler={compiler}\ndef k(x):\n return {expressions[operation]}\n" - if operation != "leaf": - assert_frozen_rejection(source, {"x": "float32"}, output_dtype) - return +def test_float_cast_arithmetic_rounding(native_artifacts, output_dtype): + compiler, jit, count = "tcc", False, 257 + source = f"# me:compiler={compiler}\ndef k(x):\n return (float(x) + 0.1) - float(x)\n" artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) if jit: @@ -515,9 +559,7 @@ def test_float_cast_arithmetic_rounding(native_artifacts, compiler, jit, count, computation = values.astype(output_dtype) constant = np.array(0.1, dtype=output_dtype) with np.errstate(all="ignore"): - expected = ( - -(computation * constant) if operation == "negation" else (computation + constant) - computation - ) + expected = (computation + constant) - computation for _ in range(2): actual = kernel.evaluate({"x": values}) np.testing.assert_array_equal(actual, expected) @@ -525,77 +567,10 @@ def test_float_cast_arithmetic_rounding(native_artifacts, compiler, jit, count, np.testing.assert_array_equal(np.signbit(actual[zeros]), np.signbit(expected[zeros])) -@pytest.mark.parametrize("case", ["math_widen", "math_condition", "math_local_widen"]) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -@pytest.mark.parametrize("count", [1, 2, 257]) -def test_math_is_outside_frozen_profile(native_artifacts, case, compiler, jit, count): - if not CORPUS.is_dir(): - pytest.skip("Native fixtures unavailable") - words = (CORPUS / f"{case}.txt").read_text().split() - input_dtype, output_dtype = words[1:3] - source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") - assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) - - -@pytest.mark.parametrize( - "case", - [ - "division_int", - "division_bool", - "division_literals", - "division_locals", - "division_loop", - "division_float32", - "division_condition", - "arithmetic_round", - "arithmetic_decimal", - "arithmetic_scalar_sub", - "arithmetic_local_round", - "arithmetic_widen", - "arithmetic_float_cast", - "arithmetic_float_nested", - "arithmetic_float_local", - "arithmetic_float_widen", - "bool_float_sum", - "bool_float_neg", - "bool_float_product", - "bool_float_difference", - "bool_float_local_neg", - ], -) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -def test_typed_arithmetic_artifact(native_artifacts, case, compiler, jit): - if not (CORPUS / f"{case}.txt").is_file(): - pytest.skip("Native arithmetic fixtures unavailable") - if jit: - control = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n return x\n") - control = blosc2.PortableKernel.from_json(control.export({"x": "float64"}, "float64"), jit=True) - if not control.has_jit: - pytest.skip(f"{compiler} backend unavailable") - words = (CORPUS / f"{case}.txt").read_text().split() - input_dtype, output_dtype = words[1:3] - rows = np.array(words[6:]).reshape(int(words[3]), 2) - source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") - if case in EXCLUDED: - assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) - return - artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) - kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - assert kernel.has_jit == jit - np.testing.assert_array_equal( - kernel.evaluate({"x": np.array(rows[:, 0], dtype=input_dtype)}), - np.array(rows[:, 1], dtype=output_dtype), - ) - - -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -@pytest.mark.parametrize("count", [1, 5, 257]) @pytest.mark.parametrize("output_dtype", ["float32", "float64"]) @pytest.mark.parametrize("operation", ["add", "subtract", "multiply"]) -def test_float_arithmetic_constant_rounding(native_artifacts, compiler, jit, count, output_dtype, operation): +def test_float_arithmetic_constant_rounding(native_artifacts, output_dtype, operation): + compiler, jit, count = "cc", True, 257 expressions = { "add": "(x + 0.1) - x", "subtract": "x - 0.1", @@ -622,11 +597,10 @@ def test_float_arithmetic_constant_rounding(native_artifacts, compiler, jit, cou np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -@pytest.mark.parametrize("count", [1, 5, 257]) +@pytest.mark.parametrize(("compiler", "jit"), [("tcc", False), ("tcc", True), ("cc", True)]) @pytest.mark.parametrize("cap", ["3", "0", "-1", "invalid"]) -def test_while_cap_policy(native_artifacts, monkeypatch, compiler, jit, count, cap): +def test_while_cap_policy(native_artifacts, monkeypatch, compiler, jit, cap): + count = 257 monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", cap) source = f"# me:compiler={compiler}\ndef k(x):\n n = 0\n while n < x:\n n = n + 1\n return n\n" artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") @@ -647,8 +621,7 @@ def test_while_cap_policy(native_artifacts, monkeypatch, compiler, jit, count, c assert kernel.evaluate({"x": np.empty(0, dtype="int64")}).size == 0 -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) +@pytest.mark.parametrize(("compiler", "jit"), [("tcc", False), ("tcc", True), ("cc", True)]) @pytest.mark.parametrize("compile_cap", ["0", "3"]) def test_while_cap_change_after_loading(native_artifacts, monkeypatch, compiler, jit, compile_cap): monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", compile_cap) @@ -674,22 +647,11 @@ def test_while_cap_change_after_loading(native_artifacts, monkeypatch, compiler, kernel.evaluate({"x": values}) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -def test_while_cap_hybrid_cleanup(native_artifacts, monkeypatch, compiler): - monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") - source = ( - f"# me:compiler={compiler}\ndef k(x):\n value = sin(x)\n n = 0\n" - " while n < x:\n n = n + 1\n return value\n" - ) - assert_frozen_rejection(source, {"x": "float64"}, "float64") - - -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -@pytest.mark.parametrize("dtype", ["int32", "int64", "float32", "float64"]) -@pytest.mark.parametrize("count", [1, 3, 257]) -@pytest.mark.parametrize("samples", [[0, 2, 3, -1], [3, 2, 0, -1]]) -def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dtype, count, samples): +@pytest.mark.parametrize( + ("dtype", "compiler", "jit"), + [("int32", "tcc", False), ("int64", "tcc", True), ("float32", "cc", True), ("float64", "tcc", True)], +) +def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dtype): monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") source = ( (CORPUS / "while_cap_chain.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") @@ -698,18 +660,22 @@ def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dt kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) if jit: assert kernel.has_jit - values = np.resize(np.array(samples, dtype=dtype), count) - expected = np.maximum(values, np.array(0, dtype=dtype)) - for _ in range(2): - np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + # Lane ordering is the actual regression boundary, not every size/backend + # combination. Keep both first-lane states in each dtype scenario. + for samples in [[0, 2, 3, -1], [3, 2, 0, -1]]: + values = np.resize(np.array(samples, dtype=dtype), 257) + expected = np.maximum(values, np.array(0, dtype=dtype)) + for _ in range(2): + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) -@pytest.mark.parametrize("dtype", ["int32", "int64", "float32", "float64"]) -@pytest.mark.parametrize("count", [4, 257]) +@pytest.mark.parametrize( + ("dtype", "compiler", "jit"), + [("int32", "tcc", False), ("int64", "tcc", True), ("float32", "cc", True), ("float64", "tcc", True)], +) @pytest.mark.parametrize("case", ["masked_local_chain", "masked_bool_chain"]) -def test_masked_local_constant_chain(native_artifacts, compiler, jit, dtype, count, case): +def test_masked_local_constant_chain(native_artifacts, compiler, jit, dtype, case): + count = 257 source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, "bool") kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) @@ -721,31 +687,39 @@ def test_masked_local_constant_chain(native_artifacts, compiler, jit, dtype, cou np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize("jit", [False, True]) +# Cap policy and post-load changes above cover all engines. These scenarios +# isolate control-flow semantics rather than crossing the same policy again. @pytest.mark.parametrize( - ("body", "expected"), + ("body", "expected", "compiler", "jit"), [ - (" n = 0\n while 0 <= n < 3:\n n = n + 1\n return n\n", 3), + (" n = 0\n while 0 <= n < 3:\n n = n + 1\n return n\n", 3, "tcc", False), ( " n = 0\n while x:\n n = n + 1\n if n == 3:\n break\n return n\n", 3, + "tcc", + True, ), ( " n = 0\n while x:\n n = n + 1\n if n == 3:\n return n\n return n\n", 3, + "cc", + True, ), ( " total = 0\n for i in range(2):\n n = 0\n while n < 3:\n" " n = n + 1\n total = total + 1\n return total\n", 6, + "tcc", + True, ), ( " total = 0\n n = 0\n while n < 3:\n n = n + 1\n m = 0\n" " while m < 3:\n m = m + 1\n total = total + 1\n return total\n", 9, + "cc", + True, ), - (" if x > 1:\n while x:\n pass\n return x\n", 1), + (" if x > 1:\n while x:\n pass\n return x\n", 1, "tcc", False), ], ) def test_while_cap_control_flow(native_artifacts, monkeypatch, compiler, jit, body, expected): @@ -761,48 +735,6 @@ def test_while_cap_control_flow(native_artifacts, monkeypatch, compiler, jit, bo ) -@pytest.mark.parametrize("compiler", ["tcc", "cc"]) -@pytest.mark.parametrize( - ("body", "output_dtype", "expected"), - [ - ("return int(x / 2) / 2", "float64", [-0.5, 0, 0, 0, 0.5]), - ("return int(x) / 2", "int64", [-1, 0, 0, 0, 1]), - ("return int(x + 0.25) / 2", "float64", [-1.5, -0.5, 0, 1, 2]), - ("if (int(x) / 2) > 0:\n return 1.0\n return 0.0", "float64", [0, 0, 0, 1, 1]), - ], -) -def test_unsupported_division_is_rejected(native_artifacts, compiler, body, output_dtype, expected): - source = f"# me:compiler={compiler}\ndef k(x):\n {body}\n" - assert_frozen_rejection(source, {"x": "float64"}, output_dtype) - - -@pytest.mark.parametrize("input_dtype", ["float32", "float64"]) -@pytest.mark.parametrize("output_dtype", ["float32", "float64"]) -@pytest.mark.parametrize("count", [1, 5, 257]) -@pytest.mark.parametrize("jit", [False, True]) -def test_nested_conversion_is_rejected(native_artifacts, input_dtype, output_dtype, count, jit): - source = "def k(x):\n return float(int(x) / 2)\n" - assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) - - -@pytest.mark.parametrize( - ("expression", "expected"), - [ - ("int(x + 0.25)", [-1, 0, 0, 2, 5]), - ("bool(x + 0.25)", [1, 0, 1, 1, 1]), - ("bool(int(x + 0.25))", [1, 0, 0, 1, 1]), - ], -) -@pytest.mark.parametrize("input_dtype", ["float32", "float64"]) -@pytest.mark.parametrize("output_dtype", ["bool", "int32", "int64", "float32", "float64"]) -@pytest.mark.parametrize("jit", [False, True]) -def test_floating_expression_casts_are_rejected( - native_artifacts, expression, expected, input_dtype, output_dtype, jit -): - source = f"# me:compiler=tcc\ndef k(x):\n return {expression}\n" - assert_frozen_rejection(source, {"x": input_dtype}, output_dtype) - - @pytest.mark.parametrize("jit", [False, True]) @pytest.mark.parametrize("expression", ["int(x)", "bool(x)"]) def test_value_cast_large_integer(native_artifacts, jit, expression): From 177f1e01499432462afdc6d5319518ae24091c30 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 21:47:33 +0200 Subject: [PATCH 19/30] Remove implicit local miniexpr checkout dependencies from tests --- tests/test_dsl_portable.py | 15 +++++++-------- tests/test_portable_artifact.py | 31 +++++++++++++++---------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 8052b20a5..474f8a9a8 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -10,12 +10,9 @@ import blosc2 from blosc2.dsl_kernel import DSLKernel -CORPUS = Path( - os.environ.get( - "MINIEXPR_PORTABLE_CORPUS", - Path(__file__).resolve().parents[2] / "miniexpr" / "tests" / "portable-dsl", - ) -) +# External conformance is opt-in; never infer an unrelated sibling checkout. +# This may point at tests/portable-dsl in CMake's pinned miniexpr fetch. +CORPUS = Path(os.environ["MINIEXPR_PORTABLE_CORPUS"]) if os.environ.get("MINIEXPR_PORTABLE_CORPUS") else None # Native CTest runs every fixture under its supported execution engines. Python @@ -41,7 +38,7 @@ TYPES = ("bool", "int32", "int64", "float32", "float64") EXCLUDED = ( set((CORPUS / "frozen-excluded.txt").read_text().split()) - if (CORPUS / "frozen-excluded.txt").is_file() + if CORPUS is not None and (CORPUS / "frozen-excluded.txt").is_file() else set() ) CASES += [f"convert_{input_type}_{output_type}" for input_type in TYPES for output_type in TYPES] @@ -73,8 +70,10 @@ def test_native_source_corpus(case, portable_validator, monkeypatch): jit = True if case.startswith("while_cap_"): monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") - if not CORPUS.is_dir(): + if CORPUS is None: pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") + # An explicit corpus is required to match the pinned profile. Do not + # silently skip missing files or misinterpret an older exclusion manifest. source = corpus_source(case).read_text() words = (CORPUS / f"{case}.txt").read_text().split() outcome, input_type, output_type = words[:3] diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 260dd88f5..efa00a2ba 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -21,12 +21,6 @@ import blosc2 -CORPUS = Path( - os.environ.get( - "MINIEXPR_PORTABLE_CORPUS", Path(__file__).resolve().parents[2] / "miniexpr/tests/portable-dsl" - ) -) -ARTIFACTS = CORPUS.parent / "portable-artifacts" SCALE = 2.0 BIAS = -1.0 @@ -321,9 +315,7 @@ def test_binding_shapes_and_storage_adaptation(native_artifacts): def test_import_is_native_only(native_artifacts, monkeypatch): - if not ARTIFACTS.is_dir(): - pytest.skip("Native fixture checkout unavailable") - artifact = (ARTIFACTS / "affine.json").read_bytes() + artifact = affine.export({"x": "float64"}, "float64") def forbidden(*args, **kwargs): pytest.fail("Python source/JSON preparation was invoked") @@ -337,11 +329,10 @@ def forbidden(*args, **kwargs): def test_export_runs_in_standalone_c(native_artifacts, tmp_path): - runner = Path( - os.environ.get( - "MINIEXPR_ARTIFACT_RUNNER", CORPUS.parents[1] / "build-portable/tests/portable_artifact_runner" - ) - ) + runner_path = os.environ.get("MINIEXPR_ARTIFACT_RUNNER") + if not runner_path: + pytest.skip("Set MINIEXPR_ARTIFACT_RUNNER to the standalone native runner") + runner = Path(runner_path) if not runner.is_file(): pytest.skip("Set MINIEXPR_ARTIFACT_RUNNER to the standalone native runner") artifact = tmp_path / "export.json" @@ -654,7 +645,8 @@ def test_while_cap_change_after_loading(native_artifacts, monkeypatch, compiler, def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dtype): monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") source = ( - (CORPUS / "while_cap_chain.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + f"# me:compiler={compiler}\ndef while_cap_chain(x):\n" + " n = 0\n while 0 <= n < x:\n n = n + 1\n return n\n" ) artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) @@ -676,7 +668,14 @@ def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dt @pytest.mark.parametrize("case", ["masked_local_chain", "masked_bool_chain"]) def test_masked_local_constant_chain(native_artifacts, compiler, jit, dtype, case): count = 257 - source = (CORPUS / f"{case}.dsl").read_text().replace("me:compiler=tcc", f"me:compiler={compiler}") + bodies = { + "masked_local_chain": " n = 0\n if x == 0:\n n = 1\n return 0 <= n < x\n", + "masked_bool_chain": ( + " flag = bool(0)\n if x == 0:\n flag = bool(1)\n" + " return bool(0) <= flag < bool(x)\n" + ), + } + source = f"# me:compiler={compiler}\ndef {case}(x):\n{bodies[case]}" artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, "bool") kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) if jit: From cac4522e4802d91e34f59014523c02d16937f5f4 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 21:52:35 +0200 Subject: [PATCH 20/30] Exercise portable artifacts without threads on WASM --- tests/test_portable_artifact.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index efa00a2ba..ac992abf2 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -232,7 +232,7 @@ def local_alias(x): @pytest.mark.parametrize("jit", [False, None, True]) -def test_artifact_backend_and_concurrent_calls(native_artifacts, jit): +def test_artifact_backend_and_repeated_calls(native_artifacts, jit): from concurrent.futures import ThreadPoolExecutor artifact = affine.export({"x": "float64"}, "float64") @@ -242,8 +242,13 @@ def test_artifact_backend_and_concurrent_calls(native_artifacts, jit): if jit is False: assert not kernel.has_jit arrays = [np.arange(600, dtype=np.float64) + i for i in range(4)] - with ThreadPoolExecutor(max_workers=4) as executor: - results = list(executor.map(lambda array: kernel.evaluate({"x": array}), arrays)) + if blosc2.IS_WASM: + # Pyodide does not provide Python threads. Keep backend selection and + # repeated-handle coverage without claiming concurrency on that host. + results = [kernel.evaluate({"x": array}) for array in arrays] + else: + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(lambda array: kernel.evaluate({"x": array}), arrays)) for array, result in zip(arrays, results, strict=True): np.testing.assert_array_equal(result, 2 * array - 1) From 2495345131cfddfdeaa324963f1548842e745896 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 21:58:38 +0200 Subject: [PATCH 21/30] Certify Windows portable semantics through native fallback --- tests/test_portable_artifact.py | 42 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index ac992abf2..7e40c22cd 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -25,6 +25,16 @@ BIAS = -1.0 +def assert_execution_mode(kernel, jit): + # Portable 0.1 certifies Windows' interpreter baseline, not its optional + # accelerator. Still execute all semantic assertions there, including + # JIT-requested fallback. On certified JIT hosts fallback must not hide a + # code-generation regression; JIT_OFF must disable it on every platform. + if jit and sys.platform == "win32": + return + assert kernel.has_jit == jit + + def assert_frozen_rejection(source, inputs, output): """Check both authoring and native import, without disguising setup errors.""" info = blosc2.validate_portable_dsl(source, inputs, output) @@ -422,7 +432,7 @@ def test_numeric_execution_shapes(native_artifacts, compiler, jit, count): source = f"# me:compiler={compiler}\ndef k(x):\n{body}" artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - assert kernel.has_jit == jit + assert_execution_mode(kernel, jit) for _ in range(2): actual = kernel.evaluate({"x": np.resize(np.array(samples, dtype=input_dtype), count)}) desired = np.resize(np.array(expected, dtype=output_dtype), count) @@ -458,8 +468,7 @@ def test_bool_output_numeric_arithmetic(native_artifacts, body, samples): {"x": "bool"}, "bool" ) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) values = np.resize(np.array([False, True]), count) expected = np.resize(np.array(samples), count) for _ in range(2): @@ -506,8 +515,7 @@ def test_bool_cast_floating_arithmetic( source = f"# me:compiler={compiler}\ndef k(x):\n{body} return {expression}\n" artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) samples = [0, 1, -1, 2] if input_dtype == "int64": samples += [2**53 + 1, -(2**53 + 1)] @@ -534,8 +542,7 @@ def test_float_cast_arithmetic_rounding(native_artifacts, output_dtype): source = f"# me:compiler={compiler}\ndef k(x):\n return (float(x) + 0.1) - float(x)\n" artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) samples = np.array( [ 2**24, @@ -575,8 +582,7 @@ def test_float_arithmetic_constant_rounding(native_artifacts, output_dtype, oper source = f"# me:compiler={compiler}\n# me:fp=strict\ndef k(x):\n return {expressions[operation]}\n" artifact = blosc2.DSLKernel.from_source(source).export({"x": "float32"}, output_dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) values = np.resize(np.array([0.1, -1, 0, 1, 16777216], dtype="float32"), count) # These expressions contain a floating literal: native compilation types # that constant from the requested floating output context. Each operation @@ -601,8 +607,7 @@ def test_while_cap_policy(native_artifacts, monkeypatch, compiler, jit, cap): source = f"# me:compiler={compiler}\ndef k(x):\n n = 0\n while n < x:\n n = n + 1\n return n\n" artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) for target in [0, 2, 3, 4]: values = np.full(count, target, dtype="int64") if cap == "3" and target == 4: @@ -624,8 +629,7 @@ def test_while_cap_change_after_loading(native_artifacts, monkeypatch, compiler, source = f"# me:compiler={compiler}\ndef k(x):\n n = 0\n while n < x:\n n = n + 1\n return n\n" artifact = blosc2.DSLKernel.from_source(source).export({"x": "int64"}, "int64") kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) values = np.array([4], dtype="int64") monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "2") with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): @@ -635,8 +639,7 @@ def test_while_cap_change_after_loading(native_artifacts, monkeypatch, compiler, # The identical source loaded under a different cap must not reuse code # compiled with the old cap from the shared JIT cache. new_kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert new_kernel.has_jit + assert_execution_mode(new_kernel, jit) np.testing.assert_array_equal(new_kernel.evaluate({"x": values}), values) monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): @@ -655,8 +658,7 @@ def test_while_chain_mixed_lane(native_artifacts, monkeypatch, compiler, jit, dt ) artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, dtype) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) # Lane ordering is the actual regression boundary, not every size/backend # combination. Keep both first-lane states in each dtype scenario. for samples in [[0, 2, 3, -1], [3, 2, 0, -1]]: @@ -683,8 +685,7 @@ def test_masked_local_constant_chain(native_artifacts, compiler, jit, dtype, cas source = f"# me:compiler={compiler}\ndef {case}(x):\n{bodies[case]}" artifact = blosc2.DSLKernel.from_source(source).export({"x": dtype}, "bool") kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) values = np.resize(np.array([0, 1, 2, -1], dtype=dtype), count) expected = np.resize(np.array([False, True, True, case == "masked_bool_chain"]), count) for _ in range(2): @@ -732,8 +733,7 @@ def test_while_cap_control_flow(native_artifacts, monkeypatch, compiler, jit, bo {"x": "int64"}, "int64" ) kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit: - assert kernel.has_jit + assert_execution_mode(kernel, jit) np.testing.assert_array_equal( kernel.evaluate({"x": np.ones(5, dtype="int64")}), np.full(5, expected, dtype="int64") ) From 302a73cdbf4fc3e547dbfc882f6911a12cb840c2 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 22:06:47 +0200 Subject: [PATCH 22/30] Preserve exact WASM integer casts and align reduction path checks --- CMakeLists.txt | 2 +- doc/reference/portable_dsl.rst | 4 ++++ tests/ndarray/test_jit_options.py | 7 ++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 07b6e0aae..d5fbb6140 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,7 +142,7 @@ endif() FetchContent_Declare(miniexpr GIT_REPOSITORY https://github.com/Blosc/miniexpr.git - GIT_TAG 7543a9592410bdfd72f7a7319343dfece55b09e4 + GIT_TAG 9a1fee2d5b7ddc9816ffdd500a244f6cb934dfc6 # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr ) FetchContent_MakeAvailable(miniexpr) diff --git a/doc/reference/portable_dsl.rst b/doc/reference/portable_dsl.rst index 1a95d5560..24e435c6d 100644 --- a/doc/reference/portable_dsl.rst +++ b/doc/reference/portable_dsl.rst @@ -68,6 +68,10 @@ interpretation. Semantic evaluation errors never retry the kernel. Artifacts enforce strict FP independently of host defaults. Host while-loop limits remain execution policy rather than artifact constants. +Windows certification covers the interpreter baseline, not optional JIT support. +Wasm32 kernels containing ``int()`` use the interpreter to preserve exact int64 +values, which the current wasm JIT cast adapter cannot represent. + The authoritative language and artifact specifications are maintained in the pinned miniexpr dependency under ``doc/dsl-spec/0.1.md`` and ``doc/dsl-spec/artifact-0.1.md``. The feature boundary is frozen; certification diff --git a/tests/ndarray/test_jit_options.py b/tests/ndarray/test_jit_options.py index 57a011376..afb00c643 100644 --- a/tests/ndarray/test_jit_options.py +++ b/tests/ndarray/test_jit_options.py @@ -391,7 +391,12 @@ def capture(self, expression, inputs, fp_accuracy, aux_reduc=None, **kwargs): values = np.arange(128, dtype=np.float64) expr = blosc2.asarray(values) * 2 + 1 assert expr.sum(jit=jit, jit_backend=backend) == (values * 2 + 1).sum() - assert calls + if blosc2.IS_WASM: + # WASM currently uses the non-prefilter reduction path. It must still + # produce correct results without introducing an elementwise wrapper. + assert not calls + else: + assert calls assert all(expression.startswith("sum(") for expression in calls) From 23a2a91a325586dd86d3dec5d21a1685b55e4c51 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 6 Oct 2026 22:08:31 +0200 Subject: [PATCH 23/30] Pin the native WASM cast certification update --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d5fbb6140..889537003 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,7 +142,7 @@ endif() FetchContent_Declare(miniexpr GIT_REPOSITORY https://github.com/Blosc/miniexpr.git - GIT_TAG 9a1fee2d5b7ddc9816ffdd500a244f6cb934dfc6 + GIT_TAG 98b2bf97e62b969aa16741e2f735ae7dce469656 # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr ) FetchContent_MakeAvailable(miniexpr) From c6ede832195a677e19111709af8630448f933ecd Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 7 Oct 2026 12:54:56 +0200 Subject: [PATCH 24/30] Add portable DSL 1.0 authoring and persistence --- CMakeLists.txt | 2 +- doc/reference/dsl_syntax.md | 35 +- doc/reference/portable_dsl.rst | 174 +++- plans/portable-DSL-1.0-implemenation.md | 499 +++++++++ plans/portable-DSL-1.0-progress.md | 1131 +++++++++++++++++++++ plans/portable-DSL-1.0-scope-inventory.md | 245 +++++ src/blosc2/b2objects.py | 155 ++- src/blosc2/blosc2_ext.pyx | 160 ++- src/blosc2/core.py | 2 +- src/blosc2/ctable.py | 261 ++++- src/blosc2/deserialization.py | 4 + src/blosc2/dict_store.py | 3 + src/blosc2/dsl_artifact_bridge.h | 57 +- src/blosc2/dsl_kernel.py | 53 +- src/blosc2/embed_store.py | 3 + src/blosc2/lazyexpr.py | 84 +- src/blosc2/msgpack_utils.py | 33 +- src/blosc2/ndarray.py | 6 + src/blosc2/portable_kernel.py | 330 ++++-- src/blosc2/portable_lazy.py | 257 +++++ src/blosc2/schunk.py | 4 +- src/blosc2/tree_store.py | 3 + tests/ctable/test_ctable_computed_cols.py | 38 +- tests/ctable/test_ctable_dsl_columns.py | 25 +- tests/ndarray/test_dsl_kernels.py | 80 +- tests/ndarray/test_lazyudf.py | 27 +- tests/test_b2objects.py | 64 +- tests/test_deserialization.py | 3 +- tests/test_dsl_portable.py | 96 +- tests/test_objectarray.py | 7 +- tests/test_portable_artifact.py | 157 +-- tests/test_portable_descriptor.py | 141 +++ tests/test_portable_descriptor_host.py | 58 ++ tests/test_portable_lazy.py | 148 +++ tests/test_portable_policy.py | 105 ++ tests/test_portable_table_export.py | 221 ++++ 36 files changed, 4117 insertions(+), 554 deletions(-) create mode 100644 plans/portable-DSL-1.0-implemenation.md create mode 100644 plans/portable-DSL-1.0-progress.md create mode 100644 plans/portable-DSL-1.0-scope-inventory.md create mode 100644 src/blosc2/portable_lazy.py create mode 100644 tests/test_portable_descriptor.py create mode 100644 tests/test_portable_descriptor_host.py create mode 100644 tests/test_portable_lazy.py create mode 100644 tests/test_portable_policy.py create mode 100644 tests/test_portable_table_export.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 889537003..bc12f9b3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,7 +142,7 @@ endif() FetchContent_Declare(miniexpr GIT_REPOSITORY https://github.com/Blosc/miniexpr.git - GIT_TAG 98b2bf97e62b969aa16741e2f735ae7dce469656 + GIT_TAG 55c882b238c4daf0523a52a44d5d84db56af24e4 # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr ) FetchContent_MakeAvailable(miniexpr) diff --git a/doc/reference/dsl_syntax.md b/doc/reference/dsl_syntax.md index ad88a6987..308ae7ae9 100644 --- a/doc/reference/dsl_syntax.md +++ b/doc/reference/dsl_syntax.md @@ -42,7 +42,9 @@ available on this path; use native function spellings such as `sin(x)`. This is separate from the existing Python-based persistence reconstruction path. Portable artifact export/import is still under development. The full native language reference and experimental portable profile live in miniexpr's -`doc/dsl-syntax.md` and `doc/dsl-spec/0.1.md`, respectively. +`doc/dsl-syntax.md` and `doc/dsl-spec/1.0.md`, respectively. The portable profile +and artifact contract are experimental drafts; see [portable artifacts](portable_dsl.rst) +for the release boundary and legacy Python persistence policy. ### Experimental portable-profile validation @@ -58,9 +60,10 @@ invokes a JIT compiler. It reports version, signature, source, and unsupported feature errors separately. Potential runtime errors, including missing returns on some paths, do not automatically invalidate a program. -The draft profile admits five numeric/Boolean dtypes with homogeneous input types -and a small function/operator set; it is narrower than the full language described -below. Validation is not a sandbox, backend-support probe, or a guarantee about +Draft 1.0 admits Boolean, standard integer widths, float32/float64 and fixed strings, +mixed operand-driven numeric typing, checked arithmetic, reductions and explicit ND context. +It is separate from the ordinary full language described below. +Validation is not a sandbox, backend-support probe, or a guarantee about arbitrary runtime data. Older native builds report `runtime_unsupported` rather than attempting Python validation. Native dependency integration for this experimental API is pending; development builds can use the local miniexpr source @@ -92,17 +95,19 @@ write. `from_json()` accepts text or bytes, not a pathname or decoded mapping. Import uses the same native JSON loader as C and does not need the author's module, function, decorators, globals, Python source reconstruction, or frontend rewrites. `evaluate()` returns a new NumPy array; `blosc2.asarray(values)` can materialize -compressed storage. This first adapter is eager, not a `lazyudf` integration. +compressed storage. `kernel.lazy()` binds explicit logical partitions for lazy +evaluation and portable persistence. Newly authored compliant DSLKernel-backed +LazyUDF saves also normalize and validate portable native recipes automatically. Python `bool`, `int`, and `float` captures infer `bool`, `int64`, and `float64`. Supported NumPy scalar types retain their dtype. Use `capture_dtypes={"SCALE": "float32"}` to explicitly specialize a capture; finite -float narrowing rounds to the chosen dtype, while overflow and integer precision -loss are rejected. Arrays (including zero-dimensional arrays), objects, unsupported +float narrowing and explicit integer-to-float conversion round to the chosen dtype; +out-of-range values reject. Arrays (including zero-dimensional arrays), objects, unsupported scalar types, unresolved names, and external callbacks cannot be captures. Export does not call the kernel or capture conversion hooks. It reuses supported authoring normalization, such as `np.sin(x)`, before native profile validation. -Strings, arbitrary globals, and unsupported functions do not become portable. +Fixed byte/Unicode scalar snapshots are supported; arbitrary objects and callbacks are not. Native-only kernels can bind existing parameters as constants explicitly: @@ -114,19 +119,19 @@ artifact = author.export({"x": "float64"}, "float64", constants={"scale": 2.0}) ``` Every source parameter requires exactly one runtime input or constant binding. -The draft profile requires homogeneous parameter dtypes, including captures; -output may have a different supported dtype. Manifest float constants preserve +Parameters and captures may have different supported dtypes; computation is +operand-driven before output conversion. Manifest float constants preserve IEEE bit patterns, including signed zero and non-finite values; integer constants are range-checked decimal strings. Comments and retained native pragmas survive parameterization. Required strict FP semantics are independent of host defaults; -compiler pragmas remain preferences, and `jit=False` disables JIT independently. +compiler pragmas remain preferences. Draft 1.0 currently always uses the typed interpreter. Runtime arrays are bound by name in any mapping order and must have the exact declared logical dtype and common shape. The adapter converts endian order, alignment, and strides, but does not broadcast arrays or silently change their -dtype. Constants broadcast in bounded native workspace. Constant-only kernels -require `evaluate({}, shape=(...))`; empty arrays validate bindings but do not -execute. `PortableArtifactError` provides `status`, `native_status`, `line`, and +dtype. Constant-only kernels require `evaluate({}, shape=(...))`; empty +elementwise arrays validate bindings without execution, whereas scalar reductions +obey their specified empty identities/errors. `PortableArtifactError` provides `status`, `native_status`, `line`, and `column` for malformed artifacts, unsupported requirements, source, binding, and evaluation failures. The artifact format is not a sandbox or an arbitrary-input numerical-correctness guarantee. @@ -143,7 +148,7 @@ conda run -n blosc2 python -m pip install -e . --no-build-isolation --no-deps \ The optional adapter links pinned yyjson only when enabled. Builds without the adapter remain usable and report `NotImplementedError` for artifact operations, without attempting Python validation/execution as a fallback. The native candidate -format and shared fixtures live in miniexpr's `doc/dsl-spec/artifact-0.1.md` and +format and shared fixtures live in miniexpr's `doc/dsl-spec/artifact-1.0.md` and `tests/portable-artifacts/`. `@blosc2.jit` auto-detects this DSL: a decorated function whose body contains an diff --git a/doc/reference/portable_dsl.rst b/doc/reference/portable_dsl.rst index 24e435c6d..b760fadaa 100644 --- a/doc/reference/portable_dsl.rst +++ b/doc/reference/portable_dsl.rst @@ -1,7 +1,13 @@ Portable DSL artifacts ====================== -Portable profile 0.1 is a conservative, versioned subset of the full native DSL. +.. warning:: + + Portable DSL 1.0 is experimental and its language and artifact specifications + are drafts. Local development integration is tested; publication, independent + numeric accuracy and release-platform certification remain pending. + +Portable DSL artifacts use the versioned native 1.0 profile. Artifacts are standalone UTF-8 JSON, executable through native miniexpr in C or Python, without reconstructing or executing Python functions on import. @@ -17,32 +23,61 @@ Python, without reconstructing or executing Python functions on import. # array([-1., 1., 3., 5.]) The result is an eager NumPy array. Bindings are named and require exact dtypes -and matching shapes. Supported dtypes are ``bool``, ``int32``, ``int64``, -``float32``, and ``float64``; all parameters and captures share one dtype, while -output may differ. Captured supported Python/NumPy scalars are snapshotted into +and matching shapes. Supported dtypes are Boolean, standard signed/unsigned +integer widths, ``float32``, ``float64`` and fixed-width bytes/Unicode strings; +parameters may have different dtypes. Captured Python/NumPy scalars are snapshotted into typed constants, never live callbacks or arrays. Integers use range-checked decimal strings; floats use exact IEEE-754 hexadecimal bits. Release boundary ---------------- +``DSLKernel.export`` and ``validate_portable_dsl`` default to draft 1.0. +Native validation of normalized source, typed captures, +fixed widths and cardinality completes before export returns JSON:: + + author = blosc2.DSLKernel.from_source("def total(x):\n return sum(x)\n") + record = author.export({"x": "int64"}, "int64", + cardinality="block_scalar") + kernel = blosc2.PortableKernel.from_json(record) + values = kernel.lazy({"x": np.arange(6, dtype="int64")}, partitions=(3,))[:] + # array([3, 12]) + +The draft admits all standard Boolean/integer widths, float32/float64, fixed byte +and Unicode strings, mixed operand-driven numeric promotion, checked arithmetic, +native math/string operations, masked ordered block reductions and logical ND +coordinates. Specify ``ndim`` for coordinate-bearing sources. Existing static +``row["column"]``, NumPy calls and string syntax reuse the full-DSL authoring +normalizers; dynamic indexing, objects and callbacks are not reconstructed. + +Raw draft source can also be checked without artifacts, arrays, execution or JIT:: + + info = blosc2.validate_portable_dsl( + "def coordinate(x):\n return x + _i0\n", + {"x": "int16"}, "int64", ndim=1, + cardinality="elementwise") + assert info["valid"] + +The core native validator checks exact numeric/string signatures and reports +native status and source line/column. String dtypes specify exact slot widths, +for example ``{"x": "U4"}``; output widths must match native inference. +``cardinality=None`` checks inferred cardinality; an explicit declaration must +match. Reserved ND symbols require an explicit covering ``ndim``. This source +validator remains available when the optional artifact JSON loader is disabled. +It checks raw native syntax, not Python normalization, globals or runtime domains. + Supported features include elementwise arithmetic, comparisons and chains, Boolean logic, local assignments, branches, bounded loops, and explicit output conversions. Boolean operands participate in numeric arithmetic as zero/one; truth conversion occurs at a Boolean output or explicit ``bool()`` call. -Excluded features include transcendental calls (including ``sin``/``cos``), -nested ``int()``/``float()`` casts, expression arguments to casts in floating-input -signatures, mixed-input dtypes, uncertified mixed computation intermediates, -integer division, float32 arithmetic inside predicates, reductions, indexing, -floating-spelled literals in integral-input signatures, -shape symbols, and external functions. A separately assigned float32 arithmetic -local can be compared after rounding. Excluded source is rejected both during -export and independently by native import; there is no Python fallback. +External functions, callbacks, object operations and unsupported dynamic indexing +are rejected during export and independently by native import; there is no Python fallback. Integer overflow, out-of-range narrowing, non-finite/out-of-range float-to-integer -casts, zero divisors, and non-default floating rounding modes are outside the -runtime contract. Callers must keep data and intermediates in the defined domains; +casts and integral zero divisors raise checked evaluation errors. The interpreter +establishes its floating-point environment and restores the caller's environment. +Callers must keep data and intermediates in the defined domains; static validation cannot prove those conditions. This is not a sandbox for untrusted source or arbitrary data. NaN payloads are not guaranteed. @@ -51,32 +86,117 @@ membership without execution or JIT. It returns validity, a status category, and optional source diagnostics. Import/export errors raise ``blosc2.PortableArtifactError`` with a stable ``status`` attribute. Missing returns, zero range steps, and loop-cap failures are runtime evaluation errors; -failed output contents are unspecified. Empty evaluations validate bindings but -do not execute the kernel. +failed output contents are unspecified. Empty elementwise evaluations validate +bindings without execution; block-scalar reductions use their specified empty identities/errors. Builds and execution -------------------- +Draft logical partitions and persistence +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a validated 1.0 kernel, ``kernel.lazy(inputs, partitions=(...))`` binds an +immutable logical C-order block grid. This grid is independent of compressed +operand chunks/blocks. Elementwise output retains the original domain; +``block_scalar`` output has one item per logical partition (the grid shape). +Scalar returns under lane-varying branches/loops, or inside loops with +lane-varying break/continue, reject as ambiguous output—even if their scalar +expressions match. Uniform reduction conditions and coherent scalar control +flow remain supported; varying loops may rejoin before a top-level scalar return. +Slices evaluate intersecting **original complete groups** before selecting result +lanes. ND coordinates retain the original domain and group origin. Rechunking the +materialized result changes storage, never reduction membership:: + + lazy = kernel.lazy({"x": stored_array}, partitions=(32, 64)) + values = lazy[1:3, ::2] + result = lazy.compute(chunks=(8, 16), blocks=(4, 8)) + lazy.save("recipe.b2nd") + reopened = blosc2.open("recipe.b2nd") + +Native JSON records, signatures and logical partitions persist through disk +carriers, frames, structured MessagePack and EmbedStore/DictStore/TreeStore, +including arithmetic expressions containing portable carriers. Operands must +have persistent references; failures preflight before overwriting destinations. +Import never reconstructs a Python function. Safe arithmetic envelopes permit +only bound-name primitive arithmetic/Boolean expressions, not calls or attributes. + +New DSLKernel-backed LazyUDF saves automatically normalize authoring and typed +captures, validate draft 1.0 natively and persist artifact/bindings/context. The +same preflight applies to frames, MessagePack, stores and nested arithmetic +references. Python UDFs and noncompliant normalized kernels reject before +destination writes; there is no source fallback. Positional scalar arguments +become typed constants, array arguments retain their parameter names, and the +current output shape and logical block partitions are preserved. A block-scalar +return cannot silently replace an elementwise LazyUDF shape: author it through +``export(cardinality="block_scalar")`` and ``PortableKernel.lazy`` +with explicit partitions instead. Storage chunk boundaries must not split the +declared uniform logical block grid. + +Historical source-backed files require explicit ``deserialize='full'``; +that policy propagates through nested references. Loaded legacy recipes are not +automatically migrated by saving. CTable legacy computed/generated DSL metadata +has the same explicit policy gate. The older table vector-column DSL API lacks +an explicit row-domain contract and cannot be silently converted to independent +rows. Use the authored row API below instead. +``msgpack_unpackb()`` also defaults to safe decoding. Active reconstruction needs +explicit ``deserialize='full'``; container readers propagate their effective +policy through nested values instead of relying on the helper's default. + +Draft native table transformers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Scalar stored columns bind to native transformers by name:: + + table.add_portable_computed_column("total", kernel, inputs={"x": "amount"}, + row_domain="independent") + table.add_portable_generated_column("stored_total", kernel, + inputs={"x": "amount"}, + row_domain="independent") + +These APIs also accept a newly authored ``DSLKernel`` directly, with explicit +output dtype. Normalization, capture snapshots and native validation finish +before column registration or destination mutation:: + + table.add_portable_computed_column( + "twice", blosc2.DSLKernel.from_source("def twice(x):\n return x * 2\n"), + inputs={"x": "amount"}, dtype="int64", row_domain="independent") + +For a fixed-shape row reduction, additionally specify +``cardinality="block_scalar"``. This row-domain declaration is required; no +grouping is inferred from the old vector-column API, table storage tiles or +append batches. + +The explicit ``independent`` domain means **each scalar row is one original +one-lane group**, not a storage block or a changing table-wide reduction. Rank-one +coordinates are row-local: shape ``(1,)``, origin ``(0,)``, ``_i0 == 0`` and +``_n0 == 1``. Block-scalar and elementwise kernels both yield one scalar per row. +Named bindings may be empty for constant/ND constructors. Appending, refreshing, +deleting rows, materializing and compact-copying retain this mapping. Artifacts, +bindings and fixed row shapes survive table disk/frame round trips, with native +validation before binding and again before saving. Fixed-width numeric/string +signatures are immutable. Fixed-shape ndarray columns also support scalar native +row reductions: each complete row is one group, its broadcast row shape is the +logical ND domain, and its origin is zero. Scalar columns can broadcast within +that row without changing coordinates. Shape/cardinality incompatibilities reject +before registration. Multi-row table partitions and vector-valued row outputs +are not implicitly admitted. The draft remains subject to conformance and +external platform release gates. + The native artifact loader is enabled by default in Python package builds. Custom builds may disable it with ``MINIEXPR_BUILD_ARTIFACT=OFF``; import then raises ``NotImplementedError``. The optional adapter uses pinned, MIT-licensed yyjson; the raw native compiler does not depend on it. -The interpreter is the baseline. TCC/CC are optional accelerators, selected with -``jit`` and native compiler pragmas. Missing accelerators can fall back to native -interpretation. Semantic evaluation errors never retry the kernel. Artifacts +Draft 1.0 uses the typed interpreter. ``jit`` and compiler pragmas are preferences +only and currently fall back to interpretation; a required-JIT artifact rejects. +Semantic evaluation errors never retry the kernel. Artifacts enforce strict FP independently of host defaults. Host while-loop limits remain execution policy rather than artifact constants. -Windows certification covers the interpreter baseline, not optional JIT support. -Wasm32 kernels containing ``int()`` use the interpreter to preserve exact int64 -values, which the current wasm JIT cast adapter cannot represent. - The authoritative language and artifact specifications are maintained in the -pinned miniexpr dependency under ``doc/dsl-spec/0.1.md`` and -``doc/dsl-spec/artifact-0.1.md``. The feature boundary is frozen; certification -on additional release platforms is tracked by CI. Broader numeric features and -lazy/container integration are deferred to a later profile. +miniexpr development sources under ``doc/dsl-spec/1.0.md`` and +``doc/dsl-spec/artifact-1.0.md``. The published dependency pin has not yet changed; +the local development override is not stock-wheel or platform certification. API --- diff --git a/plans/portable-DSL-1.0-implemenation.md b/plans/portable-DSL-1.0-implemenation.md new file mode 100644 index 000000000..50945054c --- /dev/null +++ b/plans/portable-DSL-1.0-implemenation.md @@ -0,0 +1,499 @@ +# Portable DSL 1.0 implementation plan + +Date: 2026-10-07. Status: implementation plan; all three detail decisions +below are confirmed by the user. Feature scope and earlier semantic decisions are +approved in `portable-DSL-1.0-scope-inventory.md`. This file intentionally uses +the requested filename. + +## Objective and shortest implementation path + +Deliver a public, versioned kernel contract executable from C and Python, with +portable-only new kernel persistence. Build correctness in the native +**interpreter first**. Reuse existing parsing, expression evaluation, strings, +ND support and artifact loading; do not implement a second language engine. + +Initially route every 1.0 kernel through the interpreter. Subsequently enable +only existing JIT paths whose semantics are demonstrated compatible. Missing +JIT support, including checked arithmetic, masked reductions or strings, is +never a reason to delay interpreter delivery or weaken the contract. + +Draft portable 1.0 is the sole public portable format and the authoring/validation +default. The unreleased 0.1 profile is retired, with explicit unsupported-version +rejection rather than migration or reinterpretation. Ordinary full-DSL +execution keeps its existing semantics. Miniexpr owns normative semantics, +validation and conformance; Python-Blosc2 owns normalization and host integration. + +## 0. Confirmed behavioral choices + +The user approved Q1–Q3 below. Their recommended rules are now implementation +requirements; the explanations preserve the rationale for these choices. + +### Q1. Complete the numeric type policy + +Recommend Boolean, signed/unsigned 8/16/32/64-bit integers, and float32/float64. +No float16, complex, object, datetime or structured arithmetic in 1.0. + +Proposed promotion rules: + +- Same typed operands retain their type; Boolean arithmetic uses int64 numeric + zero/one, with `/` producing float64. Boolean logical operations remain Boolean. +- Same-signedness integers widen to the wider operand. Mixed signed/unsigned + integers use the smallest signed type covering both complete operand ranges. + Reject an implicit combination for which no such type exists, notably + int64/uint64; explicit float conversion remains possible with documented loss. +- float32 with float32 stays float32; float64 dominates. Integer/float mixtures + use float32 for integers of at most 16 bits and float64 otherwise. Conversion + to float64 may round 64-bit integers; document this rather than claiming exactness. +- Weak source literals adopt an accompanying typed operand when within range. + Decimal floating literals round once to the selected IEEE type using nearest, + ties-to-even; do not require exact binary representability of `0.1`. + An out-of-range contextual literal is rejected, requiring an explicit cast or + typed constant. Literal-only integers default to int64 and floats to float64. +- Typed captured constants are strong operands, not weak literals. `int()` yields + int64, `float()` yields float64, and `bool()` yields Boolean. Explicit typed + casts, if admitted in the builtin matrix, have precisely named destination types. +- Arithmetic promotion does not define comparisons by accident: compare mixed + signed/unsigned integers exactly without lossy float intermediates. Reject any + unsupported mixed comparison explicitly in the initial matrix. + +Why approval matters: uint64 is useful, but there is no exact fixed-width signed +common type with int64. This policy chooses a diagnostic rather than implicit +float64 rounding for their arithmetic. Narrow integer arithmetic can overflow +before final widening, consistently with operand-driven computation. + +### Q2. Represent block-scalar outputs in the lazy host + +Recommend explicit result cardinality in the artifact: `elementwise` or +`block_scalar`. A standalone C block evaluation returns one value for the latter. +In Python lazy execution, expose one value per logical evaluation block, on the +block-grid shape, with row-major block ordering. Do not broadcast an actual +block-scalar return back to every element implicitly. + +A scalar reduction used inside an elementwise calculation broadcasts to its +participating lanes; that does not itself make the kernel block-scalar. +Reject paths with inconsistent return cardinality. A block-scalar partial read +indexes output blocks; an elementwise partial read evaluates original input +groups before selecting lanes. Empty input arrays have no scheduled blocks; +an explicitly evaluated empty standalone block still obeys reduction identities. + +Why approval matters: today's artifact API assumes `nitems` output values. +Silently keeping that allocation model would confuse scalar reductions with +elementwise results and obscure what a saved array's shape means. + +### Q3. Reuse the existing legacy opt-in + +Recommend user-supplied `deserialize="full"` as the legacy-kernel opt-in, rather +than adding another public switch. Normal/default safe opens must not reconstruct +legacy Python kernels. Propagate the effective policy through nested operands; +internal calls that hard-code `full` must not grant permission on the user's +behalf. If existing API defaults cannot distinguish explicit permission from an +implicit default, adjust the boundary/policy propagation before admitting legacy +execution. Portable artifacts need native validation, not Python reconstruction. + +No migration tool, automatic conversion, or legacy-save fallback is planned. +Manual adaptation guidelines are the only conversion assistance. + +## 1. Establish a small normative contract and baseline + +Files: `../miniexpr/doc/dsl-spec/1.0.md`, `artifact-1.0.md`, a machine-readable +feature/type/function matrix, and `../miniexpr/tests/portable-dsl/` fixtures. + +1. Retire unreleased 0.1 validation/loading and obsolete freeze documents. Retarget + useful encoding/numeric/masking/validation fixtures to draft 1.0, updating + independent references for operand typing and checked arithmetic. +2. Inventory actual builtin registrations and interpreter implementations once. + Classify each operation as included, existing full-DSL only, or excluded; + include arity, type combinations, result type, errors and accuracy policy. + Avoid another unbounded backend audit. Review omissions of useful existing + operations before freezing the matrix. +3. Cover arithmetic, comparisons, bitwise operations, shifts, casts, `where`, + control flow, supported math, fixed-width string operations, ND symbols and + the existing numeric/Boolean reduction families. Do not assume scalar and + reducing forms of similarly named functions share semantics. +4. Spell out floor division/remainder signs, integer power and shift overflow, + signed-zero behavior, NaNs, infinities, local type joins, short-circuiting and + evaluation order. Recommend typed locals with a statically determined join + type and rejection when no supported join exists; no dynamic dtype changes. +5. Specify loop caps and missing-return errors. Resource caps are host policy; + an error must not become a partial successful result. Preserve existing + configured caps rather than inventing a new execution sandbox. +6. Use semantic capability IDs for ND context, fixed strings and block reductions. + A declared capability is a requirement, not permission to skip validation. + +Exit: a finite matrix and example fixtures cover every admitted operation; +no undefined implementation behavior is advertised as portable semantics. + +## 2. Introduce versioned semantic compilation and conservative dispatch + +Primary files: native `dsl_portable.c`, `dsl_compile.c`, +`dsl_compile_internal.h`, `dsl_compile_support.c`, `dsl_artifact.c`. + +1. Thread a portable semantic profile through compilation and compiled program + metadata. Keep it out of process-global mutable switches. +2. Validate normalized source, resolved call targets, compiled node types, + effects, return cardinality and inferred capability requirements. Use both + source and typed-tree gates. +3. Reject callbacks, complex values, `print`, fast/contract FP, dynamic widths, + arbitrary memory access and unsupported reduction placements. A reduction + condition inside a loop is not a reduction assignment in the loop body. +4. Initially mark every 1.0 program interpreter-only. Optional JIT requests fall + back before compilation/execution of an incompatible backend. Explicit + required-JIT requests fail clearly when no certified path exists. +5. Preserve source locations through compilation and expose stable unsupported, + binding, format and evaluation error categories. + +Exit: 1.0 kernels cannot accidentally execute +under full-DSL output-context typing or an uncertified JIT. + +## 3. Implement checked operand-typed numeric interpretation + +Primary files: `dsl_compile.c`, `dsl_eval.c` and the shared native expression +typing/evaluation helpers they call. Locate shared helpers before changing them; +make changes profile-aware where ordinary DSL semantics differ. + +1. Infer every expression's computation dtype independently of declared output. + Preserve weak-literal metadata until contextual typing is complete. Insert + explicit typed conversion nodes rather than relying on output buffers. +2. Implement the approved promotion table and casts. Round float32 intermediates + at operation boundaries; prohibit implicit contraction/reassociation in the + strict profile. Test that changing output dtype does not change intermediates. +3. Implement checked integer add/subtract/multiply/negation, division/remainder, + shifts, powers and all narrowing/conversion paths actually admitted by the + matrix. Check before performing a C operation that could overflow or be UB. + Do not rely on signed overflow, invalid shifts or out-of-range C casts. +4. Check float-to-integer limits without rounded-boundary errors, including + int64/uint64 extrema. Fractional values truncate toward zero before range + acceptance is decided. Cover exact large integers without double round trips. +5. Apply numeric-zero/one Boolean arithmetic consistently to intermediates, + explicit casts and final output. Preserve fractional division before truth. +6. Propagate failures from nested expressions and final conversions. Only active + lanes execute checked operations. Short-circuit branches and `where` must not + report failures from unselected lanes; implement masked operand evaluation + rather than evaluating invalid discarded branches eagerly. +7. Document floating math domains and per-function tolerances. Use a bounded + cross-platform accuracy contract informed by existing libm implementations; + do not make bitwise transcendental equality a release requirement. +8. Keep workspace and error state per evaluation. On failure, native output is + unspecified and must not be consumed; Python raises rather than caching or + returning that block as a successful result. + +Exit: exact integer/error cases, Boolean fractions, float32 rounding, mixed +types and inactive invalid branches pass interpreter conformance on native and +WASM targets. Additional JIT arithmetic work is unnecessary for this milestone. + +## 4. Make masking and limited block reductions explicit + +Primary files: `dsl_eval.c`, `dsl_eval_internal.h`, `dsl_compile.c`, +`miniexpr_eval_reduce.c`, and ND evaluation adapters. + +1. Represent valid lanes and currently participating lanes explicitly. Track + branch, loop, break, continue and return masks; nested conditions reduce over + the mask participating at that program point. Continue excludes the rest of + that iteration but does not permanently remove a lane from the next one. +2. Top-level reductions see valid block elements; nested supported conditions see + active valid elements. Never include padded lanes or stale local storage. +3. Retain the restriction on reduction-valued assignments/returns in control-flow + bodies. Admit top-level reductions and supported conditions, including nested + `if all(...)` in Mandelbrot. Validate placements structurally, not by a string + search or a blanket ban on nonzero nesting depth. +4. Accumulate in logical C-order within a block; skip inactive lanes without + changing the order of remaining lanes. Disable parallel/tree reduction and + fast reduction shortcuts for this profile unless they give the required order. +5. Recommended operation details for the normative matrix: widen signed/Boolean + sum/prod to int64 and unsigned to uint64; keep floating sum/prod input dtype; + integer mean uses float64; floating mean uses its floating input dtype. + Checked integer accumulation applies at every step. Mean uses the specified + ordered sum followed by division by the participating count. +6. Use approved empty identities for any/all/sum; recommend prod(empty)=1 and + mean(empty)=NaN. Empty min/max errors. Recommend NaN propagation for numeric + reductions, deterministic signed-zero ties for min/max, and NaN-as-true for + truth reductions, consistently with Boolean conversion. Record these rules + explicitly and compare against existing behavior before implementation. +7. Distinguish reduced scalars from per-lane values in compiled metadata and + allocation. Broadcast scalar operands only onto participating lanes. + +Exit: Mandelbrot interpreter results survive changes in partition when its +early-exit optimization is semantically neutral; a deliberately block-dependent +example changes exactly as specified. Include partial masks, edge padding, +empty groups, continue/break/return, ordered cancellation and overflow cases. + +## 5. Extend native artifacts for context, widths and cardinality + +Primary files: `dsl_artifact.c`, `miniexpr_artifact.h`, `dsl_portable.c`. + +1. Keep standalone UTF-8 JSON, explicit schema/language versions, strict semantic + requirements, named signatures, exact typed constants and optional bindings. + Extend integer encodings for the admitted widths; retain decimal integer + strings, IEEE floating hex and Boolean values. +2. Add string descriptors with code-unit kind and fixed width. Recommend canonical + hex bytes for bytes constants and fixed-endian 32-bit code-unit hex for Unicode + constants, preserving values without assuming all slots are valid UTF-8 text. + Native buffers use host representation; adapters perform endian conversion. +3. Add result cardinality and queries for itemsize/output capacity. Introduce an + extended evaluation descriptor/API rather than silently changing the old + `me_artifact_eval()` ABI, whose output currently has `nitems` elements. +4. The extended call supplies logical shape, block origin/extent, valid-lane + description and explicit output capacity. Define `_flat_idx` in the logical + domain, `_i` as coordinates in that domain and `_n` as logical sizes. + Broadcasting inputs does not change the output coordinate domain. +5. Validate required context, dimensions, products, byte capacities, integer + conversions, ownership and overlap before evaluating. No array-sized constant + replication; no mutable context retained on a shared handle. +6. Distinguish zero-element elementwise calls from explicitly empty block-scalar + reduction calls; the latter must execute empty-group semantics. +7. Validate inferred capabilities against artifact declarations. Unknown versions + or unavailable capabilities fail explicitly, without Python reconstruction. + +Exit: a standalone C runner loads each capability fixture and evaluates supplied +buffers/context using only native code; incorrect descriptors fail deterministically. + +## 6. Admit existing fixed-width strings and logical ND kernels + +Native files: existing string implementation, `miniexpr_eval_dsl_nd.c`, +`miniexpr_eval_nd.c`, plus compile/validation and artifact descriptors above. + +1. Reuse native string operations and width inference. Specify first-NUL behavior, + full-width unterminated slots, no Unicode/bytes mixing, bytes comparison rules, + Unicode code-unit order, case-expansion truncation and static width bounds. +2. Pin the Unicode mapping/whitespace data semantics used by admitted operations; + avoid locale-dependent behavior. Separate arbitrary byte constants from the + current restriction on non-ASCII source literals used against bytes operands. +3. Check branch/local width joins and allocate the inferred return width, not a + NumPy approximation. Reject unbounded results before execution. Keep strings + interpreter-only; a host may pack fixed-width output into varlen storage. +4. Define sliced/view authoring domains explicitly: a newly constructed kernel + over a view has its declared logical domain; reading a slice of an existing + saved kernel retains that kernel's original coordinates and reduction groups. +5. Exercise zero-input constructors using ND context, sliced domains, broadcasting + and padded edge blocks. Pass context through native evaluation, not synthetic + Python callbacks or hidden global arrays. + +Exit: representative existing string/ND kernels export and execute identically +in C and Python, including widths and coordinates, with no JIT dependency. + +## 7. Python bridge, normalization and lazy execution + +Files: `src/blosc2/portable_kernel.py`, `dsl_kernel.py`, `blosc2_ext.pyx` and +associated native declarations, `lazyexpr.py`. + +1. Make export/validation APIs default to draft 1.0; reject retired versions. + select 1.0 explicitly for new persistence until the public-default transition + is documented. Import remains native-only. +2. Reuse NumPy-call, string-syntax and static-row normalization; record named + column bindings and snapshot supported typed constants, including strings. + Reject opaque captures without invoking arbitrary conversion hooks. +3. Pass context, width, shape/cardinality and evaluation errors through Cython. + Extend `PortableKernel.evaluate()` accordingly. Build interpreter-backed lazy + execution around the native handle, not around `kernel_from_source()` or exec. +4. Define a persisted logical block grid with shape, block extents, domain origin + and C-order traversal. Bind it once for reduction-bearing kernels; do not + derive it anew from current physical chunks at each evaluation. +5. Schedule complete original reduction groups for elementwise partial reads; + gather/scatter across physical storage as necessary. Elementwise-only kernels + may be tiled freely while retaining original logical ND coordinates. +6. Implement the approved block-scalar output mapping and distinguish input + domain shape from result shape. Validate immutable bound dtypes/widths and + domain requirements when reopening referenced operands. +7. Preserve author-written compiler pragmas as compiler selection preferences; + local defaults must not override them. They do not exempt the backend from + portable semantic checks or disable the agreed interpreter fallback. + +Exit: ND/string/reduction kernels run lazily through native handles, with slice +results and reduction grouping unchanged after storage rechunking. + +## 8. Enforce portable-only kernel persistence end to end + +Files: `b2objects.py`, `lazyexpr.py`, `deserialization.py`, relevant open/from-frame +entry points and CTable computed/generated-column serializers. + +1. Trace every DSL-kernel save/load route: structured B2Objects, older LazyUDF + metadata paths, embedded/nested lazy operands, CTable columns and frames. + Use one encoder/validator and one native decoder rather than divergent formats. +2. Preflight normalized kernel portability and typed bindings before creating or + replacing the destination. Save the artifact plus explicit operand references, + logical context/partition and container format discriminator. +3. Remove broad exception-driven fallback to source-based persistence for DSL + kernels. A portability failure must survive to the caller with location/reason. + Do not redefine ordinary non-kernel LazyExpr persistence by accident. +4. On load, distinguish portable, legacy and malformed records using metadata + only. Validate portable artifacts natively before binding/execution; never + reconstruct Python from a portable record, even after validation fails. +5. Apply explicit legacy opt-in consistently, including recursive operands. + Review current hard-coded `deserialize="full"` calls in `b2objects.py` and + effective defaults in `deserialization.py`; do not allow an internal reopen + to bypass the user's selected policy. +6. No automatic migration, conversion command or source-rewrite-on-open. + Document manual changes and explicit legacy use. Container/data-source + portability is separate from standalone kernel portability; demonstrate C + execution of the stored artifact with supplied bindings, without promising + C support for every Python remote/container protocol. + +Exit: all new DSL save routes reject excluded features; round trips preserve +native-only execution and block context; legacy files require opt-in everywhere. + +## 9. Optional JIT eligibility, not a JIT implementation project + +After interpreter milestones, build a conservative eligibility predicate over +the typed program and backend capabilities. Its default answer is no. + +- Allow only operation/type/context combinations already shown to implement 1.0 + semantics. Include semantic version, dtype/width and relevant policy in cache + identity so cached full-DSL programs cannot masquerade as 1.0 programs. +- Keep checked integers, unsupported casts, strings, ND/reduction combinations + and uncertain masking on the interpreter until individually certified. +- Optional JIT unavailability or unsupported lowering selects interpreter before + execution. Do not retry an executed, failing kernel using another backend; + arithmetic/domain failures remain failures. +- Required-JIT remains an explicit host diagnostic mode: report unsupported + acceleration instead of falsely claiming JIT execution. +- Retain the Windows interpreter baseline and WASM exact-integer fallback. + No target is required to acquire a new code generator for 1.0. + +Exit: eligible kernels match interpreter semantics; ineligible kernels execute +correctly by default and report a reason when JIT is explicitly required. + +## 10. Bounded conformance and integration verification + +Do not cross-product dtype, operation, backend, length and storage dimensions. +Put the detailed semantic corpus in miniexpr and keep approximately 100–200 +orthogonal Python portable tests, restructuring existing cases as necessary. + +Native fixtures cover exact boundary/error cases for each checked operation; +representative promotion/literal pairs; rounding and inactive-lane behavior; +Mandelbrot and masked reductions; ND origins/padding; string encodings/widths; +artifact errors, capacity and ownership; immutable concurrent handles. + +Python integration covers normalization and captures, standalone-native import, +native error propagation, each save route, legacy opt-in, rejection without +partial destination replacement, partial reads/rechunking, scalar-block shapes, +default fallback and explicit required-JIT failure. Use exact expected values +for integers rather than a NumPy overflow reference. Floating expectations use +the specified rounding/order or function-specific tolerance. + +Normal Python tests use the published/pinned miniexpr and self-contained fixtures. +Keep external corpus/C-runner checks opt-in through the established environment +variables; never infer a sibling checkout at runtime. + +Run targeted tests after each milestone, native sanitizers for masks/capacity/ +arithmetic, then full native/Python CI including Windows and WASM at integration. +Use the `blosc2` conda environment for Python, builds and tests. Smoke-test an +isolated wheel built without local native overrides. Test artifact-disabled +builds report missing capability cleanly. Update the published native pin only +after the required native code is available; no unpublished-checkout dependency. + +## Delivery order and completion checklist + +### Autonomous execution and durable handoff + +Follow the milestones in order, recording progress and verification after each. +The autonomous implementation target is completion of the beta-readiness gates, +not collection of external beta feedback or automatic freezing of the contract. + +- Repository scope covers the necessary changes in Python-Blosc2 and miniexpr. + Follow each repository's local instructions. Preserve unrelated user work; + avoid unrelated refactors and whole-language JIT audits. +- Prioritize interpreter correctness and use pre-execution JIT fallback. Do not + spend a milestone's effort implementing optional acceleration. +- Continue through routine implementation, build and test failures without + asking permission. Resolve ordinary details from approved semantics, verified + implementation behavior and the normative matrix. +- Pause only for unresolved semantic decisions, scope contradictions or external + blockers such as unavailable credentials. Record the precise blocker and + proposed resolution; continue independent unblocked milestones where possible. +- Commits, pushes, publishing and other repository/release state changes require + explicit user authorization consistent with repository instructions. This plan + does not itself grant those permissions. Run available local verification; + record any CI checks blocked by missing authorization or credentials. +- Keep a durable handoff in `plans/portable-DSL-1.0-progress.md`. Create it at the + start of implementation and update it after every milestone and before any + interruption or context handoff. Include completed checklist items, changed + files in both repositories, exact verification commands/results, unresolved + failures, decisions and their approval status, and the next concrete action. + Record relevant native revision/build/pin information so another session can + reproduce the environment. Do not store credentials or secrets. +- On resume, read this plan, the scope agreement, the progress file and current + working-tree changes before editing. Do not assume an interrupted command + completed or discard partial work. Reverify uncertain results. +- At completion, summarize delivered behavior and remaining external release + gates. Do not claim CI success, beta feedback or release publication without + evidence. + +Checklist status is **local implementation**, not release/platform certification; +authoritative commands/results are in `portable-DSL-1.0-progress.md`. + +1. [x] Finish finite normative draft type/arity/domain/tie/NaN/reduction/Unicode + matrix (Q1–Q3 confirmed), with analytic/exact-reference fixtures. Independent + function-specific cross-platform libm accuracy bounds remain a release gate. +2. [x] Add versioned compilation/interpreter-only 1.0 dispatch, including public + core-only source validation and extended ND/cardinality/string-width descriptor. +3. [x] Complete checked operand-typed numeric semantics and active-lane errors. +4. [x] Complete reduction masks/cardinality, including rejection of divergent + scalar return sites and preservation of finished-lane iterator values. +5. [x] Extend artifact ABI/schema/context and native runner/descriptor fixtures. +6. [x] Admit bounded fixed strings and logical ND descriptors through native code. +7. [x] Integrate Python native handles and immutable logical-block scheduling. +8. [x] Enforce validated saves/native loads and recursive explicit legacy opt-in + across the implemented disk/frame/container/table routes, with safe MessagePack + defaults. No source fallback; table row-domain modes are explicit. + New authored DSLKernel-backed LazyUDF saves automatically normalize and export + validated 1.0 artifacts, with typed scalar/capture snapshots and named array + bindings. Preserve the elementwise domain and current uniform logical block + grid; reject true block-scalar kernels with explicit portable API guidance. + This is authoring preflight, not automatic migration of loaded legacy carriers. + Explicit independent-row table APIs accept DSLKernel authoring plus output + dtype/cardinality directly; old vector-column APIs have no row-domain contract + and reject persistence with a specific ambiguity diagnostic. +9. [x] Conservative interpreter-only dispatch and pre-execution optional JIT + fallback; required-JIT rejects unsupported acceleration explicitly. +10. [ ] Complete release verification gates: + - [x] Compact local Python/native conformance and full local ASAN/UBSAN. + - [x] Isolated native-development override wheels and artifact-disabled smoke. + - [x] Draft/API docs build; changed portable page free of diagnostics. + - [ ] Windows/Linux/WASM external execution and independent per-function + accuracy certification (no universal ULP bound inferred from host libm). + - [x] Publish/pin the native revision and verify a stock-pin macOS ARM64 wheel. + - [ ] Complete the remaining stock-pin platform wheel matrix. + - [ ] Clean whole-tree docs gate: existing installation/list-table, missing + image/generated-documentation/reference warnings outside portable changes. + +Each step must land as a usable tested increment; reduction and string fixtures +can be prepared while numeric interpretation is implemented, without introducing +unapproved semantics. If a useful existing feature cannot fit the finite matrix, +raise the specific incompatibility rather than silently shrinking scope. + +Release documentation includes normative native specs, Python API/persistence +guidelines, manual legacy adaptation, standalone C examples, and corrected DSL +reference statements. These checks establish readiness for the beta milestone +below; freezing the public 1.0 contract additionally requires beta feedback. +Do not tie completion to broad JIT acceleration, migration utilities or new +remote-storage protocols. + +## Release milestone: miniexpr beta and portable 1.0 release candidate + +Once the implementation checklist passes, move miniexpr into beta with portable +DSL 1.0 as a **release-candidate specification**. Interpreter conformance across +supported platforms, native artifact loading, and working persistence integration +are the entry gates. Limited JIT coverage does not block beta: correct +pre-execution interpreter fallback is sufficient. + +Keep the two release promises distinct: + +- **Miniexpr beta:** implementation and APIs are approaching release readiness; + broader use should exercise integration, usability and performance. +- **Portable DSL 1.0, once frozen:** a public compatibility commitment independent + of whether the miniexpr implementation is still in beta. + +During beta, exercise real persisted kernels and cross-language C/Python loading, +including ND context, fixed-width strings, block reductions, partial reads and +legacy opt-in. Resolve contract ambiguities and compatibility issues found in +that feedback before freezing 1.0. Clearly label release-candidate artifacts and +their compatibility status; do not present draft artifacts as already covered +by the final 1.0 guarantee. + +- [ ] Enter miniexpr beta after the implementation and verification gates pass. +- [ ] Publish portable 1.0 as a release-candidate specification during beta. +- [ ] Review real-world persistence and cross-language feedback; resolve blockers. +- [ ] Freeze and publish the portable 1.0 compatibility contract after that review. + +Further JIT acceleration remains an independent optimization, not a prerequisite +for either beta entry or contract freeze. diff --git a/plans/portable-DSL-1.0-progress.md b/plans/portable-DSL-1.0-progress.md new file mode 100644 index 000000000..210c24b13 --- /dev/null +++ b/plans/portable-DSL-1.0-progress.md @@ -0,0 +1,1131 @@ +# Portable DSL 1.0 implementation progress + +## Published dependency update — latest status + +The user authorized committing/pushing miniexpr and committing the Python +implementation plus dependency pin. Native implementation is published on +`Blosc/miniexpr` main at `55c882b238c4daf0523a52a44d5d84db56af24e4`. +Python-Blosc2 `CMakeLists.txt` pins that exact revision. Earlier notes saying the +published pin is unchanged are historical and superseded by this update. + +A fresh stock wheel (no local source overrides) built successfully using that +revision. Both `FETCHCONTENT_SOURCE_DIR_MINIEXPR` and +`FETCHCONTENT_SOURCE_DIR_BLOSC2` are empty in its CMake cache, and the fetched +native checkout reports the pinned SHA. The isolated installed wheel passed all +seven portable test modules: **183 passed, 17 external opt-in skips** (200 cases). +The editable development installation was not replaced. + +Verification commands used `conda run -n blosc2 python -m pip wheel . --no-deps +--no-build-isolation --wheel-dir /portable-1-published-pin-wheel +--config-settings=build-dir=/portable-1-published-pin-build`, followed +by a separate `pip install --no-deps --target /portable-1-published-pin-smoke` +and pytest with the editable finder excluded only in that subprocess. Wheel SHA256: +`434b4cb6fdb1e5fa11dd088a749a42359ed798242e8de05cef69710d642c9d87`. + +This verifies the published-pin macOS ARM64 wheel, not Linux/Windows/WASM +certification or beta readiness. Unrelated user work remains unstaged. + +## CURRENT STATUS — authoritative handoff (supersedes all older sections below) + +### Unreleased portable 0.1 retired; draft 1.0 is the public default (2026-10-07) + +The user's corrected release scope is implemented in both repositories. Portable +0.1 was never released and has no compatibility obligation. The public story is +experimental/draft portable 1.0 versus legacy Python-specific persistence, whose +active reconstruction requires explicit `deserialize='full'`. No automatic +migration or reinterpretation occurs. Ordinary full-DSL semantics are unchanged. + +- `DSLKernel.export`, its authoring helper and `validate_portable_dsl` default to + 1.0. Removed the Python 0.1 dtype/call allowlists, scalar encoder and export + branch. Old native builds report unavailable draft support instead of exposing + their retired loader; the published native pin remains unchanged. +- Native `dsl_portable.c` now contains only typed draft validation. The artifact + loader rejects unsupported schema/language versions before interpreting their + envelope as 1.0, then strictly validates context/signature/capabilities/constants. + Removed the old source filter, full-profile artifact compilation, tiled evaluator + state and mask/ND rejection branch. `me_artifact_eval` remains a useful rank-zero + numeric elementwise call adapter to descriptor evaluation, not format support. +- Portable compilation no longer inherits full-DSL `ME_DSL_FP_MODE` defaults; + it uses the strict parsed draft default and still rejects explicit non-strict + source. Ordinary full-DSL environment policy remains intact. +- Removed obsolete freeze/language/artifact draft documents and the 18-entry + frozen-excluded manifest/category/rejection tests. Retargeted the native corpus + and runners to interpreter-first 1.0. Separate full-DSL audit cases remain. + Five corpus reference files now reflect operand-driven float32 arithmetic, + explicit float64 casts and integral Boolean-negation zero, rather than output + context typing. Native encoding, schema, ownership, binding and descriptor + regressions remain and now load 1.0. +- The Python host suite retargets the former frozen boundary to actual arithmetic + admission/results and checked overflow, preserving all 200 orthogonal cases. + Mixed captures and NumPy normalization now execute; the old rejected string + capture case now verifies a typed Unicode snapshot. Optional JIT requests always + assert interpreter fallback, never pretend `has_jit=True`. External corpus/runner + tests use explicit environment paths only and now exercise portable artifacts. +- User-facing Python/native API docs and specifications describe draft 1.0 only, + with a prominent experimental notice and no invented beta/platform claim. + Implementation plan/scope current decisions record retirement. Historical + verification below and unrelated user files/logs are retained. + +Verification so far, all commands in conda `blosc2`: + +- Complete native CTest: **325 passed**; complete ASAN+UBSAN CTest: **325 passed**. + The count drops from 343 solely because the 18 obsolete frozen rejection gates + were removed; useful arithmetic/masking/encoding/validation fixtures remain. +- Seven portable Python modules: **183 passed, 17 external skips = 200 cases**. + Explicit external native corpus/runner integration: **176 passed, zero skips** + across `test_portable_artifact.py` and `test_dsl_portable.py`. +- Full default Python suite: **11280 passed, 55 skipped**, 36.35s, after the final + bridge/validator changes and retained Unicode capture regression. +- Native warning-enabled rebuild retains existing unused helpers/aggregate + initializers/unknown SIMD pragmas and duplicate-library linker warnings. No new + warning in the retirement validator/artifact/runner changes or sanitizer finding. + +Remaining release gates are unchanged: independent function/platform accuracy, +Windows/Linux/WASM and stock-published-pin wheel certification, authorized native +publication/pin update, and clean whole-tree documentation. Local development +override wheel results are not stock-published-pin evidence. + +### Final retirement verification and file handoff + +- Rebuilt editable extension with explicit local miniexpr and C-Blosc2 overrides. + After isolated wheel installs, rechecked source import path and active descriptor + support; the editable install remains in use. Published pin/dependencies unchanged. +- Isolated local-native-override wheel: all seven portable modules **183 passed, + 17 external skips**, 1.80s. Smoke subprocess removed only its own editable finder, + asserted the imported package was beneath its isolated target and used `-n 0`. + Wheel SHA256 `16a514a8063601c4a4eb1ce1ebfe1a0b13e1421c8e4cbb1e32126424127c290f`. + This is not stock-published-pin, platform or published-wheel certification. +- Separate artifact-disabled local-override wheel built/installed only to another + isolated target. Ordinary array arithmetic and draft numeric/string/ND source + validation pass; retired source version rejects and artifact loading explicitly + reports `NotImplementedError`. No installed editable replacement. +- Final native and ASAN+UBSAN suites: **325/325** each, 9.08s / 9.23s. Final + incremental builds report only the existing duplicate `-lm`/static-library + linker warning; no sanitizer error. Earlier warning-enabled full rebuild retains + the pre-existing warnings documented above, not a warning-free whole-tree claim. +- Ruff lint and format checks pass on all four Python files changed by retirement; + Cython compiled successfully in editable and both wheel builds. Both repositories + pass `git diff --check`. +- Sphinx exits successfully; neither changed portable/native-syntax Python page has + a diagnostic. Whole-tree output contains **931 WARNING / 4 ERROR lines**, including + existing installation list-table, NumPy/reference/image/generated-stub issues. + This remains an incomplete clean docs gate, not a release acceptance claim. +- Final callsite/version audit over source/tests/docs/examples/build/CI files finds + no retired portable support/default/freeze links; explicit 0.1 rejection tests + and numeric literals remain. Moved the prior numeric audit to + `../miniexpr/plans/numeric-audit-0.1-history.md` as a factual archive, out of public + specifications. Unrelated untracked user work and user logs are untouched. +- Final additive regression verifies an actual retired envelope (no `context`, + `core-scalar`, `scalar-per-element`) rejects specifically as unsupported version, + not as malformed 1.0. The containing diagnostics case reran **1 passed**; + lint/format and both whitespace checks passed again. No extra test-count grid. + +Retirement files (in addition to preserved earlier 1.0 implementation): + +- Python: `src/blosc2/dsl_kernel.py`, `portable_kernel.py`, `blosc2_ext.pyx`, + `dsl_artifact_bridge.h`; `tests/test_portable_artifact.py`, `test_dsl_portable.py`; + `doc/reference/portable_dsl.rst`, `dsl_syntax.md`; current plan, scope and progress. +- Native: `src/dsl_portable.c`, `dsl_artifact.c`, `dsl_compile.c`, `miniexpr.h`, + `miniexpr_artifact.h`, `dsl_portable_types.h`, `functions.c`; + `tests/CMakeLists.txt`, `test_dsl_portable_validation.c`, `test_dsl_artifact.c`; + both portable runner/README directories, affine JSON and five arithmetic/Boolean + reference fixtures; draft 1.0 specs and native DSL syntax/usage docs. + Removed `doc/dsl-spec/{0.1,artifact-0.1,draft-history-0.1}.md` and + `tests/portable-dsl/frozen-excluded.txt`; preserved historical numeric audit as above. +- Reproducible logs/builds/wheels are under the approved temporary root with + `retire-` prefix: editable, full/target/external Python, native/sanitized builds + and tests, wheel build/install/tests, disabled build/install/smoke and docs. + +## PRIOR IMPLEMENTATION HANDOFF — historical; superseded by retirement above + +Final parent-session check: the seven portable Python modules passed again +(**183 passed, 17 external skips**, 200 cases); whitespace checks passed in both +repositories. Local implementation milestones 1–9 are complete as recorded in +the implementation checklist. Remaining milestone 10 gates are external release +certification and pre-existing whole-tree documentation issues, not a claim that +miniexpr has entered beta or portable 1.0 is frozen. + +- Parent scope correction implemented: compliant newly authored DSLKernel-backed + LazyUDF saves/frames/stores/MessagePack/nested references now automatically + normalize and export validated native 1.0 recipes. Blanket rejection was too + strict and is removed. Shared `portable_from_lazyudf` preflight preserves typed + captures/scalar bindings, parameter-named array refs, output domain and existing + uniform logical block partitions. Block-scalar return/elementwise shape conflicts + reject with explicit `PortableKernel.lazy` guidance; chunk boundaries that split + that uniform block grid reject rather than inventing grouping. Loaded legacy + recipes are marked and remain nonmigrating, explicitly full-policy-only on load. +- CTable independent-row APIs accept DSLKernel authors directly with explicit + output dtype and reduction cardinality; authoring validates before registration. + Old vector-column APIs remain ambiguous about row-domain/grouping and now report + this specific reason and the usable authored independent-row route. Restored + add-square/loop/where/object/table roundtrips replace obsolete blanket-rejection + tests. Unsupported effect/callback and legacy saves preserve destinations. + Latest scope-correction verification: **11280 Python / 343 native / 343 + ASAN+UBSAN**, isolated development wheel **458 passed, 17 external skips**. + The compact portable boundary is unchanged at **200 cases**. Details below. +- Public validation closure is now implemented: core `me_validate_portable_dsl` + explicitly dispatches 1.0; new `me_validate_portable_dsl_ex` carries exact string + output width, logical ND rank and asserted/inferred cardinality. It uses the + existing profile parser/compiler with JIT OFF and no input buffers/callbacks, + execution, Python reconstruction or optional yyjson/artifact dependency. + Python `validate_portable_dsl(..., language_version='1.0', ndim=..., + cardinality=...)` routes to that API, preserving frozen 0.1 and reporting native + status/source locations. Actual local editable extension was rebuilt. +- The finite 1.0 matrix/spec now has normative draft arities/types, checked + combinatoric domains/width limits, ~ semantics, rounding/ties, NaN/zero rules, + ordered/empty/coherent reductions, Unicode 15.0.0 case data/exact whitespace + sets and string index/width rules. Independent analytic/hexadecimal references, + adjacent float steps, exact fused cancellation, all integer factorial limits, + canonical/alias math and explicit exceptional-value fixtures supplement—not + replace—the older same-host libm regression matrix. No universal accuracy + bound or external platform certification is invented. Implementation checklist + now distinguishes completed local milestones 1–9 from specific release gates. +- Independent correctness review found and fixed two boundary bugs (finished-loop + lane iterator clobbering and NumPy signed-minimum capture conversion warnings). + Post-review suites still pass: **11280 Python / 343 native / 343 ASAN+UBSAN**; + portable boundary remains **200 cases (183 passed, 17 external skips)**. + Both surfaced contract concerns are now **resolved**: typed structural validation + rejects divergent block-scalar returns as ambiguous output, preserving coherent + scalar control flow; MessagePack defaults to safe with explicit recursive caller + policy propagation. Post-fix isolated native-override wheel also passes the + entire 200-case portable boundary. See contract-follow-up addendum below. +- Installed conda `blosc2` editable extension uses the explicit local miniexpr + development override. Published native pin is unchanged. Local 1.0 tests run, + not skip. No commits, publishing or external platform certification performed. +- Native typed interpreter, draft schema/signature/constants, fixed strings, + ND descriptors, masked reductions, logical lazy partitions, partial reads, + streaming output/rechunking and safe portable persistence are connected. +- Added explicit independent-row CTable native computed/generated APIs: + `add_portable_computed_column` and `add_portable_generated_column`, requiring + `row_domain='independent'`. Each row is an original one-lane group; ND shape + (1,), origin (0,) are row-local, not global table indices. Append, refresh, + deletion and compacted save do not change these semantics. Fixed-shape ndarray + row inputs now also support scalar native reductions, with scalar/array input + broadcasting inside the complete row group, persisted immutable row shapes and + zero row-local ND origin. Table records carry artifact JSON, bindings and + row-domain metadata and load natively, never Python functions. Multi-row table + partitions/vector-valued output modes are explicitly unsupported; the native + standalone/lazy APIs still support the full logical-partition contract. +- Added explicit `DSLKernel.export(..., version='1.0', cardinality=..., ndim=...)`. + Reuses existing static row/string/NumPy call normalization and capture token + parameterization; typed signed/unsigned/float/byte/Unicode snapshots. Native + load validates normalized source and output widths/cardinality before export + succeeds. Default export remains frozen 0.1; no migration on import. +- Full default Python suite after resolving persistence regressions and a real + carrierless RemoteArray policy bug: **11280 passed, 55 skipped**. The policy + helper now handles objects whose `schunk` is None without losing effective + recursive policy. Previously failing old new-save expectations now verify + rejection of noncompliant kernels before destination writes; compliant authored + save roundtrips were restored by the parent scope correction above. Old + explicit-full load fixtures remain. +- Local native-development wheel built and isolated import/native reduction smoke + passed, explicitly **not a stock published-pin wheel result**. Artifact-disabled + development wheel built and isolated ordinary arithmetic/missing-loader smoke + passed. Editable install was not replaced by either isolated wheel install. +- Added finite native float32/float64 operation result-type/domain fixtures for + 26 numeric libm operations and six exceptional domains. These compare host + libm within explicit 8-epsilon relative bounds; independent cross-platform ULP + certification is still an external release-matrix gate, not claimed complete. + +Current changed inventory (including preserved earlier edits): native sources/tests +and draft specs listed in historical inventory below; Python bridge +`blosc2_ext.pyx`, `dsl_artifact_bridge.h`, `portable_kernel.py`, `portable_lazy.py`, +`dsl_kernel.py`, `ctable.py`, `deserialization.py`, `b2objects.py`, `lazyexpr.py`, +`schunk.py`, `core.py`, `ndarray.py`, `msgpack_utils.py`, `embed_store.py`, +`dict_store.py`, `tree_store.py`; `doc/reference/portable_dsl.rst`; portable tests, +legacy policy/persistence tests and affected table/ndarray/objectarray tests. +Most recent native closure: `src/dsl_portable.c`, `src/miniexpr.h`, and the +public-validation/independent-reference test functions. Keep unrelated user +files and all existing uncommitted changes. + +Table-copy/materialization and native signature/shape/dependency revalidation on +save and load are now implemented. Executed fixtures include static row/NumPy +normalization, typed numeric/Unicode snapshots, scalar/ND constructors, scalar +and vector-row append/extend/refresh, fixed-string row outputs, compact-copy, +deletion, materialization, disk/frame round trips, no Python reconstruction, and +malformed-artifact rejection preserving an existing destination. The portable +Python boundary remains **200 cases: 183 passed, 17 opt-in external skips**. +Native default suite **343/343 passed** and complete ASAN/UBSAN suite +**343/343 passed**; the two targeted artifact/interpreter sanitizer tests also +passed separately. Added descriptor initializers produce no new test warnings; +the full sanitizer build exposes older aggregate-initializer warnings in existing +nonportable test files. No sanitizer error was reported. + +Exact remaining gates: independent function-specific/platform accuracy matrix, +Windows/Linux/WASM and stock-published-pin wheel CI, publication/native pin +authorization, and a clean whole-tree docs gate (existing installation list-table +parsing errors, missing tutorial images, duplicate/generated documentation and +unrelated reference warnings remain). Development docs build succeeds and the +portable page has no reported diagnostics. Local draft integration is tested; +do not claim all beta/platform acceptance criteria complete. Final post-edit +Python/lint/docs/wheel results are recorded in the addendum below. + +### Exact implementation file inventory (preserved earlier work included) + +Python tracked changes: +`doc/reference/portable_dsl.rst`, `src/blosc2/b2objects.py`, +`src/blosc2/blosc2_ext.pyx`, `src/blosc2/core.py`, `src/blosc2/ctable.py`, +`src/blosc2/deserialization.py`, `src/blosc2/dict_store.py`, +`src/blosc2/dsl_artifact_bridge.h`, `src/blosc2/dsl_kernel.py`, +`src/blosc2/embed_store.py`, `src/blosc2/lazyexpr.py`, +`src/blosc2/msgpack_utils.py`, `src/blosc2/ndarray.py`, +`src/blosc2/portable_kernel.py`, `src/blosc2/schunk.py`, +`src/blosc2/tree_store.py`, `tests/ctable/test_ctable_computed_cols.py`, +`tests/ctable/test_ctable_dsl_columns.py`, `tests/ndarray/test_dsl_kernels.py`, +`tests/ndarray/test_lazyudf.py`, `tests/test_b2objects.py`, +`tests/test_deserialization.py`, `tests/test_dsl_portable.py`, +`tests/test_objectarray.py`. +New implementation files: +`src/blosc2/portable_lazy.py`, `tests/test_portable_descriptor.py`, +`tests/test_portable_descriptor_host.py`, `tests/test_portable_lazy.py`, +`tests/test_portable_policy.py`, `tests/test_portable_table_export.py`. +Durable task files: this progress document, the approved implementation plan and +scope inventory. Generated extension/build/wheel/log products are not source +implementation files; unrelated user untracked files remain untouched. + +Miniexpr tracked changes: +`CMakeLists.txt`, `src/miniexpr.h`, `src/dsl_portable.c`, +`src/dsl_artifact.c`, `src/dsl_compare.c`, +`src/dsl_compile.c`, `src/dsl_compile_internal.h`, +`src/dsl_compile_support.c`, `src/dsl_eval.c`, `src/dsl_eval_internal.h`, +`src/dsl_jit_runtime_host.c`, `src/dsl_jit_runtime_internal.h`, +`src/dsl_jit_runtime_nonhost.c`, `src/dsl_parser.c`, `src/dsl_parser.h`, +`src/functions.c`, `src/functions.h`, `src/miniexpr.c`, +`src/miniexpr_artifact.h`, `src/miniexpr_internal.h`, +`tests/test_dsl_artifact.c`, `tests/test_dsl_portable_validation.c`. +New miniexpr implementation files: +`doc/dsl-spec/1.0.md`, `doc/dsl-spec/artifact-1.0.md`, +`doc/dsl-spec/features-1.0.json`, `src/dsl_portable_expr.c`, +`src/dsl_portable_expr.h`, `src/dsl_portable_fp.h`, +`src/dsl_portable_types.c`, `src/dsl_portable_types.h`, +`src/dsl_semantic_profile.h`, `tests/test_dsl_portable_checked.c`, +`tests/test_dsl_portable_interp.c`, `tests/test_dsl_portable_types.c`. + +### Authored-save scope correction — latest verification (authoritative) + +This section supersedes all verification records below. No native implementation +changes were needed for authored-save conversion; the existing native 1.0 parser, +compiler and artifact validation remain authoritative. All checks used conda +`blosc2` and explicit development overrides where building wheels. + +- Full Python default suite: **11280 passed, 55 skipped**, 32.62s. Restored native + roundtrips for add-square, static loops/where, slices, named bindings, DictStore + operands, ObjectArray, captures/scalar constants, frames and nested arithmetic. + Complete logical-block reductions plus ND coordinates and sliced reads are + checked against independent expected values, retaining original partitions. +- Seven portable modules: **183 passed, 17 external skips = 200 cases**; tests + extend existing orthogonal cases rather than introducing another parameter grid. + Unsupported print/callback stores preserve old entries on Embed/Dict/Tree routes; + unsupported disk saves and loaded legacy recipe saves preserve destinations. +- Complete native suite: **343 passed**, 8.34s. Complete ASAN/UBSAN suite: + **343 passed**, 8.49s, `ASAN_OPTIONS=symbolize=0`. No sanitizer failures. +- Development-override wheel built and installed only to an isolated target; + isolated subprocess asserted package path and removed only its editable finder. + Portable modules plus b2objects/ObjectArray/ndarray DSL/CTable DSL/computed-column + modules: **458 passed, 17 external skips, 1 network deselected**, 4.22s. + Wheel `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `a4255253709584e7a1ccb69d585dc9ed6cc8a80d100b59a7bed0f6234f31e1c9`. + This is not stock-pin/platform certification. Editable import path and descriptor + availability were subsequently rechecked and preserved. +- Ruff check/format passed for the 10 Python files touched by this correction; + repository whitespace check passed. Sphinx build succeeds with **391 whole-tree + warnings**, including missing generated API stubs; the portable page itself has + no reported diagnostic. The clean whole-tree docs gate remains incomplete. +- Authoring route: `portable_from_lazyudf` in `src/blosc2/portable_lazy.py`, shared + through `b2objects.encode_b2object_payload` and LazyUDF save/frame. CTable APIs + accept DSLKernel plus `dtype`, explicit `row_domain='independent'` and optional + `cardinality='block_scalar'`; the approved plan and portable docs record this. + No legacy source fallback/migration or changed elementwise-to-scalar shape. +- Logs under the approved temporary root: `author-save-python.log`, + `author-save-portable.log`, `author-save-native.log`, `author-save-sanitized.log`, + `author-save-wheel-build.log`, `author-save-wheel-tests.log`, `author-save-docs.log`. + Wheel and isolated target directories: `author-save-wheel`, `author-save-wheel-smoke`. +- Local corrected scope is fulfilled. External function-specific accuracy, + Windows/Linux/WASM, authorized publication/pin and stock-wheel certification, + and clean whole-tree docs remain pending. No commits/push/publication occurred. + +### Public validator and finite-reference closure — earlier verification (superseded) + +These results supersede the earlier verification records below. All Python, +build and test commands used conda `blosc2`; native overrides were explicit. + +- Rebuilt the installed editable extension with the documented local miniexpr + and C-Blosc2 overrides. Rechecked `blosc2.__file__` resolves to this repository's + `src/blosc2/`, descriptor support is available, and 1.0 Unicode source validation + succeeds after both isolated wheel installations. Published pin unchanged. +- Full default Python suite: **11280 passed, 55 skipped**, 42.87s. +- Final complete native CTest: **343 passed**, 8.73s. Final complete ASAN/UBSAN + CTest: **343 passed**, 8.86s, `ASAN_OPTIONS=symbolize=0`. These include the + extended validator and independent analytic/exact reference fixtures, including + the final floating-power NaN exceptions. No sanitizer errors or new warnings + in the modified validator/interpreter tests were reported. +- Isolated development-override wheel: **183 passed, 17 external skips** across + the seven portable modules = **200 cases**. No new parametrized cases or local + 1.0 skips; validation assertions extend the existing authoring test. Wheel: + `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `39dc5fcd00867991a06dc14426829a305bccc3fa581224e5bb8dfc71b0e096bf`. + Isolated subprocess removed only its own editable finder, asserted wheel import + path, and ran pytest with `-n 0`. This is **not stock-pin certification**. +- Separately built/installed isolated artifact-disabled wheel: core frozen 0.1 + and draft 1.0 numeric/string/ND source validation passed, missing ND context + rejected, ordinary arithmetic passed, optional artifact import explicitly + rejected. Plain source validation does not require the optional JSON loader. +- Ruff check and format pass for all **27 changed/new Python files**; both + repositories' `git diff --check` pass. Finite feature JSON parses successfully. +- Sphinx HTML build succeeds with **142 whole-tree warnings**; no diagnostic + against `doc/reference/portable_dsl.rst` or the updated validator docstring. + Existing generated/reference/installation/image diagnostics are not a clean + documentation gate and remain recorded as outstanding. +- Logs under `/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode/`: + `closure-editable-build.log`, `closure-python.log`, `closure-native-build.log`, + `closure-native-ctest.log`, `closure-sanitized-build.log`, + `closure-sanitized-ctest.log`, `closure-wheel-build.log`, + `closure-wheel-tests.log`, `closure-disabled-build.log`, `closure-docs.log`. + Isolated wheel directories are `closure-wheel`, `closure-wheel-smoke`, + `closure-disabled-wheel`, and `closure-disabled-smoke`. +- Local requested implementation closure is complete. Remaining release gates: + independent function-specific/platform accuracy acceptance, Windows/Linux/WASM + execution, authorized native publication/pin and stock-pin wheel matrix, and + clean whole-tree docs. No beta-ready claim, publication, dependency change or + destructive repository operation was performed. + +### Earlier local verification addendum (superseded by closure results above) + +- `conda run -n blosc2 pytest -q`: **11280 passed, 55 skipped**, 38.69s, + after final row-shape/broadcast/width fixes. Subsequent test-helper/comment + cleanup reran ndarray DSL and CTable DSL tests: **138 passed**, 1.88s. +- Seven compact portable test modules: **183 passed, 17 skipped** = **200**; + the 17 skips are 16 external frozen-0.1 corpus cases and one external artifact + runner. No local draft 1.0 test skips. Frozen 0.1 self-contained artifact tests + continue to execute, with default export still 0.1. +- Complete native interpreter CTest: **343 passed**. Complete ASAN/UBSAN CTest: + **343 passed**, 43.90s, `ASAN_OPTIONS=symbolize=0`. Targeted artifact and + portable interpreter sanitizer pair: **2 passed**, 0.87s. No test failure or + sanitizer error; the existing nonportable aggregate initializer warnings noted + above do not indicate a new implementation warning. +- Ruff lint and format: all **27 changed/new Python files passed**. Cython is + validated by actual wheel compilation, not Ruff's Python parser. Both repos' + `git diff --check` passed. +- Final native-development override wheel: + `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `1d88d169e4a2af3df0747bdd5002deca019fab77aeac9b32aa3d40a1bb0f05a9`. + Built with explicit local miniexpr/C-Blosc2 source overrides and artifacts ON. + Installed into a separate temporary target; removed the editable import finder + only inside the smoke subprocess and asserted the loaded wheel package path. + The **entire 200-case portable suite** ran against that isolated wheel: + **183 passed, 17 external opt-in skipped**, 1.08s, including typed authoring, + table vector/string/constructor persistence and lazy descriptor tests. + This is **not** a published-pin/stock-wheel platform certification result. +- Separate artifact-disabled development wheel built and isolated smoke passed: + ordinary array arithmetic works, availability query is false, portable loader + explicitly raises `NotImplementedError`. Editable development install remained + untouched and was rechecked for the active descriptor ABI afterwards. +- Final incremental Sphinx HTML build succeeded with **117 whole-tree warnings**; + no diagnostic names `doc/reference/portable_dsl.rst` after correcting its new + heading underline. Earlier clean full build exposed existing installation + list-table parsing errors and unrelated missing-image/reference warnings. + A successful non-`-W` build is not a clean whole-tree documentation gate. + +Logs/builds/wheels remain under the approved temporary root +`/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode/`: +`portable-1-full-python-final.log`, `portable-1-comments-regression.log`, +`portable-1-native-ctest.log`, `portable-1-sanitized-all-ctest.log`, +`portable-1-sanitized-all-build.log`, `portable-1-docs-final.log`, +`portable-1-final-wheel/`, `portable-1-final-wheel-smoke/`, +`portable-1-disabled-wheel/`, `portable-1-disabled-smoke/`. +The supported local draft integration is implemented and tested. External +platform/independent-accuracy/stock-pin gates remain unexecuted, and no dependency +pin, publishing or repository state-changing git action was performed. + +### Independent correctness review addendum (2026-10-07) + +Scope: independently read repository instructions, approved scope/plan and current +handoff; inspected profile-aware expression typing/conversions, reduction masks, +scalar returns, constant decoding, artifact width/capability/cardinality and +descriptor checks, Python typed capture/export, lazy original-group reads and +persistence/policy propagation. This is focused local review, **not complete +assurance**, a whole-JIT audit or external platform certification. + +Concrete fixes: + +- `../miniexpr/src/dsl_eval.c`: portable `for` evaluation no longer zeroes the + iterator local in inactive lanes while another lane continues. Before the fix, + `for i in range(x): ...; return i` on `[2, 4]` returned `[0, 3]`, rather than + `[1, 3]`. Preserve full-DSL behavior outside the versioned portable profile. + Native regressions cover exhausted ranges, lane-specific break, and runtime + rejection of undefined iterators for empty ranges. +- `src/blosc2/portable_kernel.py`: determine integral capture magnitude with a + Python integer rather than `abs()` in the NumPy fixed-width dtype. Capturing + `np.int64(INT64_MIN)` as float64 formerly raised an overflow RuntimeWarning + (an exception under the suite's warning policy). Captured conversion and + immutable snapshot now have an authoring regression. +- Merged host boundary checks into existing portable cases: masked scalar return + through a reduction local with the first lane invalid, all-invalid sum identity, + tampered scalar cardinality/string output width rejection, and reverse/empty + elementwise partial reads of `x + sum(x)` preserving exactly the original group. + No test counts expanded, tests suppressed or failures relabeled. + +Findings from the initial review — resolved by the contract follow-up below: + +1. **Divergent block-scalar returns:** native import accepts + `s = sum(x); if x < 0: return s + 1; return s` as `block_scalar` (normalized + multiline DSL). Input `[-1, 2]` returns `1`: the first subset wrote `2`, then + the remaining subset overwrote the one scalar output with `1`. + `dsl_compile.c` recognizes the reduced local as uniform without accounting for + varying return control; `dsl_eval.c` writes each scalar return to the same slot. + The follow-up now rejects divergent scalar returns structurally; it neither + invents a selection rule nor reinterprets cardinality. +2. **Implicit legacy permission at the low-level MessagePack boundary:** + `msgpack_unpackb(payload)` in `msgpack_utils.py` deliberately defaults to FULL. + A handcrafted extension-43 legacy `lazyudf` record reconstructed a `LazyUDF` + without passing `deserialize='full'`. Safe-mode paths retain propagation and + rejection; this is not a demonstrated safe-mode bypass. The helper's documented + compatibility exception conflicted with the plan's explicit-opt-in wording. + The follow-up now defaults this boundary to safe and propagates explicit-full + permission through nested values, without a legacy/default-full fallback. + +Exact post-fix verification (conda `blosc2`, local native development override): + +- Rebuilt native interpreter, sanitizer build and Python editable extension using + the existing local miniexpr/C-Blosc2 FetchContent overrides; published pin unchanged. +- Full Python `pytest -q`: **11280 passed, 55 skipped**, **31.17s**, after final + boundary-test edits. Seven portable modules: **183 passed, 17 external skipped**, + **1.79s**, still exactly **200 cases**. +- Full native CTest: **343/343 passed**, **8.75s**. Full sanitizer CTest with + `ASAN_OPTIONS=symbolize=0`: **343/343 passed**, **8.91s**. No sanitizer error; + incremental review builds reported no warnings. A new exploratory test initially + assumed preassigned loop iterators were admissible; native compilation rejects + this existing syntax restriction. Replaced that assumption with the actual + supported empty-range/undefined-local boundary, not a change in language policy. +- Ruff lint and format checks pass for all four Python files changed by review; + both repos' `git diff --check` passes. No post-review wheel/docs rebuild claimed. +- Logs: approved temporary root, `portable-1-review-{native-build,native-ctest, + sanitized-build,sanitized-ctest,editable,full-python,portable-python}.log`. + +External beta gates remain independent function/platform accuracy certification, +Linux/Windows/WASM and stock-published-pin wheel CI, publication/pin authorization, +and clean whole-tree docs. The semantic/policy findings above were additional +local acceptance concerns and have now been resolved, as recorded below. +No commits, pushes, pin changes or publishing performed; unrelated files preserved. + +### Contract follow-up — both independent-review concerns resolved (2026-10-07) + +- `../miniexpr/src/dsl_compile.c` validates block-scalar return coherence after + static type/provenance joins converge. Reject returns under typed lane-varying + branches or loop bounds/conditions, including scalar returns in loops whose + lane-varying break/continue can split the return mask. Diagnostic: + `ambiguous block-scalar output: return under lane-varying control or loop flow`. + No first/last scalar selection and no output-cardinality reinterpretation. + Uniform reduced-scalar conditions/coherent scalar loops still compile; varying + loops may rejoin before a single top-level scalar return. Nested loop flow + rejoins locally rather than tainting unrelated outer returns. +- This intentionally conservative check also rejects matching scalar return + expressions inside a varying branch/loop: equality alone does not establish + coherent group control flow. The exact excluded patterns are documented in + `../miniexpr/doc/dsl-spec/1.0.md` and `doc/reference/portable_dsl.rst`. + Native and merged Python cases exercise divergent reduced-local returns, + varying range/while/elif, lane-varying break/continue, and accepted reduction- + controlled/coherent-loop and post-rejoin scalar returns. +- `src/blosc2/msgpack_utils.py`: public and private decoding boundaries default + safe. Explicit-full extension decoding passes its effective policy recursively + into frames, structured references, NumPy object values and sets. Audited all + source callsites: SChunk VLMeta/getall, EmbedStore metadata, ObjectArray and + BatchArray already pass their owning carrier's policy; proxy reads explicitly + pass safe. No internal call inherits the newly changed public default as a + substitute for caller policy. Locally constructed/full containers retain + their existing effective policy; safe-opened containers remain safe. +- Merged legacy MessagePack checks into the existing portable policy case: + default/safe handcrafted extension-43 legacy payloads, both direct and nested + in extension-46 object arrays, reject with a monkeypatched source-reconstruction + failure hook never called. Explicit-full decoding reconstructs those same + values intentionally. Existing local/full metadata and container tests pass + without adding default-full fallback or weakening any safe assertion. + +Final exact local verification: + +- `conda run -n blosc2 pytest -q`: **11280 passed, 55 skipped**, **34.60s**, after + final boundary checks. Seven portable modules: **183 passed, 17 external + skipped**, **3.10s**, exactly **200 cases**. Targeted descriptor/policy pair: + **10 passed**, **2.15s**. +- Rebuilt interpreter and sanitizer native builds. Complete CTest: + **343/343 passed**, **45.29s**; complete ASAN/UBSAN CTest with + `ASAN_OPTIONS=symbolize=0`: **343/343 passed**, **45.59s**. No sanitizer error or + new compiler warning; both incremental links reported the existing duplicate + `-lm`/`libminiexpr.a` warning. +- Rebuilt editable install with explicit local miniexpr/C-Blosc2 overrides; + rechecked installed source import and descriptor ABI after isolated wheel work. +- Built native-override wheel + `blosc2-4.14.2.dev0-cp311-abi3-macosx_26_0_arm64.whl`, SHA256 + `a0874703e46863b67040fbc096aa007c734e56f5f1b26da1fb903caa670cf649`. + Installed to a separate temporary target; removed the editable finder only in + the smoke subprocess and asserted the package path. Complete portable suite + against the isolated wheel: **183 passed, 17 external skipped**, **1.52s**, + using `-n 0` so worker startup cannot reintroduce editable imports. This is + local-development override evidence, **not stock-published-pin certification**. +- Ruff lint/format pass for all six Python implementation/test files changed by + independent review/follow-up; both repos' `git diff --check` passes. +- Sphinx command exits successfully; portable page has no diagnostic. Whole-tree + output includes **818 WARNING / 4 ERROR diagnostic lines**, including existing + installation list-table errors and unrelated reference/image issues. This is + not a clean docs gate, and no unrelated docs cleanup was attempted. +- Approved temporary-root logs use `portable-1-contract-` prefix: native/sanitized + build and CTest, editable, full/portable Python, wheel build/install/test and + docs. Wheel and target directories: `portable-1-contract-wheel/` and + `portable-1-contract-wheel-smoke/`. + +No remaining local semantic question from these two findings. External beta +gates remain independent function/platform accuracy certification, Linux/Windows/ +WASM and stock-published-pin wheel CI, authorized publication/native pin change, +and clean whole-tree docs. No backend expansion, commits, pushes or publication. + +## HISTORICAL RECORDS — superseded status/next-actions, retained verification history + +All inventory, pending-work and skip statements below describe earlier sessions. +They are not current handoff instructions. Use CURRENT STATUS above and its final +verification addendum for resume decisions. + +## Target and approved decisions + +Implement `portable-DSL-1.0-implemenation.md` through beta readiness. All Q1–Q3 +are approved: standard signed/unsigned integer widths and operand promotion; +explicit block-scalar cardinality/grid output; explicit `deserialize="full"` +legacy opt-in, without migration tooling. Interpreter correctness comes first. +No final contract freeze or publication is implied by completing implementation. + +## Starting state + +- Python-Blosc2: `23a2a91a`; no tracked modifications on entry. +- Miniexpr: `98b2bf9`; no tracked modifications on entry. +- Numerous unrelated untracked files exist in Python-Blosc2; preserve them. +- Native repository instructions: `../miniexpr/AGENTS.md` (K&R C style, minimal + changes, relevant tests, no new dependencies). +- Python commands must run through `conda run -n blosc2`. + +## Verification log + +- Baseline: `conda run -n blosc2 pytest tests/test_portable_artifact.py + tests/test_dsl_portable.py -q`: **159 passed, 41 skipped**. Skips include opt-in + external corpus/runner tests; this does not certify portable 1.0. +- Native build directory: + `/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode/miniexpr-portable-1-interpreter`. + Configured with `conda run -n blosc2 cmake -S /Users/faltet/blosc/miniexpr -B + -DMINIEXPR_ENABLE_TCC_JIT=OFF -DMINIEXPR_USE_SLEEF=OFF + -DMINIEXPR_USE_ACCELERATE=OFF -DMINIEXPR_BUILD_SHARED=OFF + -DMINIEXPR_BUILD_EXAMPLES=OFF -DMINIEXPR_BUILD_BENCH=OFF + -DMINIEXPR_BUILD_ARTIFACT=ON`; built with `--parallel 8`. +- First native suite with promotion helper: **341/341 passed**. Latest suite + adds checked-arithmetic tests: `conda run -n blosc2 ctest --test-dir + --parallel 8 --quiet --output-log /portable-1-native-ctest.log` + exited successfully. Current suite has 342 tests. +- Checked arithmetic compiled independently with `conda run -n blosc2 cc + -std=c99 -Wall -Wextra -Werror -fsanitize=address,undefined + -fno-sanitize-recover=all -I /Users/faltet/blosc/miniexpr/src + /Users/faltet/blosc/miniexpr/src/dsl_portable_types.c + /Users/faltet/blosc/miniexpr/tests/test_dsl_portable_checked.c + -o /portable-1-checked-sanitized`; executing it **passed**. + No compiler/sanitizer warnings. Full CMake build emits its existing duplicate + static-library/`-lm` linker warning; no new compiler warning was observed. +- Draft matrix audit: all **87** distinct registered `functions.c` builtin names + are covered, plus four DSL intrinsics (`int`, `float`, `bool`, `range`); no + duplicate matrix names. This is coverage of the inventory, not certification. +- `git diff --check` passes in both repositories. + +## Milestones + +1. In progress: implemented draft schema and finite builtin inventory; concrete + platform/domain/tolerance certification remains (not a schema design blocker). +2. Integrated: profile-aware typed interpreter and draft artifact 1.0 acceptance + for validated implemented capabilities; standalone 0.1 validator stays frozen. +3. Integrated: numeric operations/casts/comparisons/lazy operands and static local + joins. Remaining work is the finite conformance/platform matrix. +4. Integrated: ordered masked reductions, padded lanes, scalar cardinality, + empty standalone blocks and evaluation-local caches/definition masks; expand + returned-lane/control-flow/concurrency certification fixtures. +5. Integrated: extended artifact descriptor ABI, typed signatures/constants and + immutable width/cardinality/capability/context validation. +6. Integrated: fixed strings, dynamic integral indices/bounded replacement, and + logical ND coordinates; platform/endian/Unicode conformance remains. +7. In progress: conditional Python bridge and standalone descriptor routing done; + logical lazy scheduling, partial reads and compiled Python 1.0 tests remain. +8. Pending: all persistence routes and legacy policy enforcement. +9. Integrated: interpreter-only optional-JIT fallback and explicit required-JIT + unsupported diagnostic; frozen/full profiles preserve their original behavior. +10. Pending: complete conformance/platform/wheel/docs verification. + +## Findings to retain + +- `functions.c` registers `log` differently depending on `ME_NAT_LOG`; the + portable profile must bind natural log explicitly, independent of build flags. +- Existing artifact eval assumes elementwise `nitems` output and skips empty + calls. Extended descriptors are required for block-scalar and empty identities. +- Existing `b2objects.py` internally hard-codes `deserialize="full"` on operand + opens. Effective user policy must propagate instead. +- Full-DSL compilation uses shared expression helpers; approved operand typing + must be profile-aware, not an unversioned global change. + +## Next action + +Continue with static local joins, remaining numeric functions and mask conformance, +then extended artifact descriptors/cardinality/context. +Do not enable 1.0 acceptance until the typed-tree/evaluator actually enforces the +contract. Current public runtime APIs still accept only 0.1; staged semantics are +isolated from frozen/full-DSL execution. No semantic blocker identified yet. + +## Continuation increment: typed interpreter integration + +- Added internal `dsl_compile_program_profile()` and `private_compile_profile_ex()`; + the original entry points retain full-DSL behavior. Profile metadata is per + program/parser state, not global. The 1.0 raw-tree path bypasses full-DSL numeric + promotion, optimization, bytecode and output-context node retyping. +- Parser leaves preserve exact decimal integer magnitudes (through uint64), weak + literal identity and direct float32 literal rounding. Unary literal signs are + resolved without executing arithmetic. Typed pass inserts checked conversion + nodes, including final output conversion only after operand-driven computation. +- Added `dsl_portable_expr.[ch]` operating on the existing `me_expr` tree and + checked helpers. Core arithmetic, shifts/bitwise, powers, exact mixed integer + comparisons, int/float/bool casts and lazy truth/where now execute natively. + Unsupported operations still reject in this **internal staged** path. +- DSL assignment/return and condition masks route through the checked evaluator; + skipped lanes do not execute invalid arithmetic. New locals infer independently + of output dtype. Existing locals still use their first inferred type; approved + static join inference remains to implement. +- Optional JIT requests are forced to interpreter before IR construction in 1.0. + Print is rejected. No public 1.0 dispatch or persistence changed yet. +- Reconfigured/rebuilt the native build above; `tests/test_dsl_portable_interp` + passed. `conda run -n blosc2 ctest --test-dir --parallel 8 --quiet + --output-log /portable-1-native-ctest.log` passed (343 tests). +- Additional changed native files: `functions.[ch]`, `miniexpr.c`, + `dsl_compile.c`, `dsl_compile_internal.h`, `dsl_jit_runtime_internal.h`, + `dsl_eval.c`, `dsl_semantic_profile.h`, `dsl_portable_expr.[ch]`, + `tests/test_dsl_portable_interp.c`, and the CMake source list. +- This initial increment's tests were subsequently expanded and explicitly made + assertion-enabled even in Release builds; use the latest verification below. + +## Continuation increment: ordered masked reductions and numeric coverage + +- Profile-aware DSL parsing retains `//=` instead of lowering through floating + `floor(a / b)`. Expression tokenization and comparison lowering recognize `//`; + full-DSL tokenization/compound-assignment behavior stays unchanged. Checked + integer floor division/remainder, float floor division/divisor-sign remainder, + and distinct truncation-sign builtin `fmod` now execute through typed nodes. +- Added explicit builtin identity dispatch for the staged unary libm family, + aliases, selected binary libm functions, abs/square/sign and real-only + conj/real/imag. Calls use float32 libm entry points for float32 computation and + float64 otherwise; no external callback is executed. Natural-log binding is + independent of `ME_NAT_LOG`. `rint` ties-to-even does not use host rounding mode; + `round` keeps native half-away ties. Scalar fmin/fmax zero ties are explicit. +- sum/prod/mean/min/max/any/all typed nodes now reduce current participating lanes + in logical order, with checked integer accumulation and float32 operation + boundaries. Empty identities/errors, numeric NaN propagation and min/max zero + ties are implemented. A per-expression evaluation-local reduction cache + broadcasts without quadratic repeated reduction work for normal expressions; + it is never stored on a compiled/shared handle. +- Expanded native integration coverage: exact int64/uint64 literals and + comparisons, contextual literal errors, arithmetic/shift/power extrema, + operand-vs-output widths, narrowing, NaN truth/comparisons, libm typing, + conditional invalid arithmetic, nested masked reductions, loop early exit, + returned lanes, continue, empty identities and ordered cancellation/overflow. +- Added profile guards to `dsl_jit_runtime_host.c`, + `dsl_jit_runtime_nonhost.c`, and both execution paths in `dsl_eval.c` so no + uncertified cached/backend function can execute for a 1.0 program. +- Staged compilation rejects external callbacks, print, non-strict FP, ND symbols + without the unfinished explicit descriptor, unsupported calls/strings, nested + reductions and reduction-valued short-circuit/where alternatives needing + additional expression masks. These are **temporary implementation diagnostics**, + not approved scope removals. Local dtype-changing reassignment rejects clearly + rather than silently narrowing to the first assignment. Static joins remain + required. Inconsistent return cardinality now rejects. +- Additional changed native files: `dsl_parser.[ch]`, `dsl_compare.c`, + `dsl_jit_runtime_host.c`, `dsl_jit_runtime_nonhost.c`, and updated draft `1.0.md`. + +### Latest reproducible verification + +Let `T=/private/var/folders/tb/7hwq2y354bb_68xwxjwjwwlr0000gn/T/opencode`. +All commands ran through the required `blosc2` environment. + +- `conda run -n blosc2 cmake --build $T/miniexpr-portable-1-interpreter --parallel 8` + passed; log `$T/portable-1-build.log`. Normal build has only the previously + observed duplicate-library linker warning. +- `conda run -n blosc2 ctest --test-dir $T/miniexpr-portable-1-interpreter + --parallel 8 --quiet --output-log $T/portable-1-native-ctest.log`: **343 passed**. +- Integrated sanitizer build configured with: + `conda run -n blosc2 cmake -S /Users/faltet/blosc/miniexpr + -B $T/miniexpr-portable-1-sanitized -DMINIEXPR_ENABLE_TCC_JIT=OFF + -DMINIEXPR_USE_SLEEF=OFF -DMINIEXPR_USE_ACCELERATE=OFF + -DMINIEXPR_BUILD_SHARED=OFF -DMINIEXPR_BUILD_EXAMPLES=OFF + -DMINIEXPR_BUILD_BENCH=OFF -DMINIEXPR_BUILD_ARTIFACT=ON + -DCMAKE_BUILD_TYPE=Debug + '-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-sanitize-recover=all -Wall -Wextra'`. + `conda run -n blosc2 cmake --build $T/miniexpr-portable-1-sanitized + --target test_dsl_portable_interp --parallel 8` and + `conda run -n blosc2 $T/miniexpr-portable-1-sanitized/tests/test_dsl_portable_interp` + **passed**, with no sanitizer findings. The warning-enabled shared-source build + emits numerous existing unused-function/initializer warnings; no warnings came + from the new expression module. Log `$T/portable-1-sanitized-build.log`. +- Independently: `conda run -n blosc2 cc -std=c99 -Wall -Wextra -Werror -I src + -c src/dsl_portable_expr.c -o $T/portable-1-expr-warning-check.o` **passed**. +- Python regression command from the baseline rerun: **159 passed, 41 skipped**. + It uses the installed/pinned native dependency, not sibling-checkout inference; + it does not verify staged 1.0 or newly edited native code. +- Latest `git diff --check` passes in both repositories. + +### Handoff and true remaining gates + +No unresolved approved-semantic conflict or external blocker was encountered. +The implementation is coherent/tested but **not beta-ready**. No public 1.0 +validator/artifact acceptance, Python native bridge/persistence transition or pin +update was made; no release/publication/platform-CI result is claimed. + +Next concrete work: + +1. Add static numeric local joins (do not reintroduce output-context typing), + finish remaining matrix operations (`fma`, `ldexp`, `logaddexp`, e/pi and + combinatorics), per-function domains/tolerances and bound/range conversion + conformance. Review the FP environment/nearest-rounding requirement explicitly. +2. Extend expression masks for reduction-valued lazy operands if admitted; + complete Mandelbrot/partition/padding/concurrent-handle fixtures and cardinality + joins. Existing program eval still broadcasts scalar outputs into nitems slots + and does not execute an empty standalone block; do not certify that old ABI. +3. Implement extended native artifact schema/API with capacities, valid lanes, + logical context and block-scalar outputs. Only then admit strings/ND and public + 1.0 validation/loading, retaining frozen 0.1 dispatch. +4. Continue Python bridge, immutable logical block scheduling, all persistence + routes and effective explicit legacy opt-in, then the remaining beta gates. + Do not update the published native pin to unpublished local code. + +## Further continuation: static joins and strict FP environment + +- Implemented bounded fixed-point numeric local inference using discovery trees + that are discarded, followed by rebuilding against stable slot dtypes. Numeric + return paths also join when output dtype is inferred. Storage conversion is + inserted after RHS operand-driven computation, not before it. Frozen/full DSL + continues its original single-pass compilation. For-variable collision checks + are preserved while permitting discovery-pass rebuilds. +- Added `dsl_portable_fp.h`: staged compilation and expression evaluation save + the caller thread's floating environment, establish nontrapping IEEE nearest/ + ties-even, then restore rounding and exception flags on success/failure. No + shared handle or global semantic switch stores this state. +- Added local reassignment/loop/return-join tests and compilation/evaluation under + FE_UPWARD with pre-existing FE_INVALID, checking nearest arithmetic/literals and + exact caller environment restoration. Target `test_dsl_portable_interp` rebuilt + and **passed** in the ordinary native build. Integrated sanitizers/full suite + will be rerun after the next dependent increment. +- Supersedes the earlier temporary "local type joins not implemented" diagnostic + and nearest-rounding TODO. Remaining operation, masks, descriptor and host gates + are unchanged; public 1.0 acceptance remains closed. + +## Further continuation: remaining numeric operations and lazy reduction masks + +- Added typed fma/fmaf (explicit fusion), integral-exponent ldexp with checked int + bounds, stable logaddexp including equal infinities, exact e/pi constants, and + checked integral fac/ncr/npr. Combinatorics retains the promoted integral dtype; + negative operands or k>n error. Binomial factors cancel before multiplication, + so e.g. uint64 ncr(67,33) succeeds without spurious intermediate overflow. +- Found native pure FUNCTION0 token value 40 collides with logical-not token 40. + Portable tokenization preserves zero-argument builtin identity without that + token flag; frozen/full behavior is not changed. +- Reduction-valued where alternatives and short-circuit RHS now have explicitly + derived participating masks, with evaluation-local child contexts/caches and + ownership cleanup. Discarded arithmetic/reductions are not executed. Generated + synthetic short-circuit assignments are admitted structurally while retaining + the prohibition on user reduction assignments/returns in control-flow bodies. +- Compact integration cases cover checked factorial, exact large binomial, + explicit fused cancellation, ldexp, infinite logaddexp, constants, masked + where/and reductions and nested alternative masks. Ordinary native target + `test_dsl_portable_interp` rebuilt and **passed**; sanitizer/full rerun follows. +- Supersedes the temporary reduction-valued lazy-operand diagnostic. All numeric + builtin names now have staged implementations (complete domain/accuracy corpus + and finite per-operation matrix details are still certification work). + +## Further continuation: descriptors, empty groups, definition state and fixtures + +- Added internal `dsl_eval_program_portable()` with a byte-capacity descriptor, + valid-lane mask, single-item block-scalar output and explicit empty scalar-block + execution. Reduction provenance propagates through locals; lane-dependent + condition/range/loop-flow assignments demote uniformity. Empty reductions use + the real zero-lane group, with a separate dependency slot for scalar locals. + Elementwise zero-item calls still skip execution. Legacy eval ABI is unchanged. +- Added per-call/per-lane local initialization masks to staged numeric evaluation; + participating uninitialized reads error, lazy discarded reads do not. Loop + variable writes define only active lanes. Range arguments now use checked int64 + conversions instead of legacy unchecked float/uint64 casts. +- Added public extended **descriptor ABI** (not language/schema 1.0 loading): + `me_artifact_eval_ex`, buffer widths/capacities, logical-context/mask fields, + cardinality and itemsize queries. Frozen 0.1 rejects unavailable context/masks. + Product/address extent/alignment/overlap checks precede execution; old ABI is + preserved. Tests cover undersized/misaligned/overlapping buffers, bad descriptor + versions, overflow and empty calls with no data/output buffers. +- FP environment setup now also verifies gradual float32/float64 underflow, + rejecting flush-to-zero/denormals-are-zero environments without arch-specific + or global mutable switches. Updated native language/artifact drafts. +- Compact fixtures added for scalar output sentinels, empty sum/mean/min, local + scalar provenance, definition masks, invalid range bounds, lane-dependent + cardinality, Mandelbrot-style break/partition/padding, and concurrent shared + handle calls with distinct caller rounding/exception flags. +- Descriptor/artifact and initial empty-group increment passed all **343** native + tests; targeted interpreter/artifact ASAN/UBSAN tests both passed. Subsequent + definition/range/FP/fixture/cardinality edits are being verified in the next + full/sanitizer rerun. No public 1.0 acceptance or native pin update occurred. + +- Latest integrated rerun after definition/range/gradual-underflow fixtures and + lane-dependent cardinality demotion: full **343/343 native tests passed**; + targeted artifact/interpreter ASAN/UBSAN **2/2 passed**; the typed expression + module still compiles with `-Wall -Wextra -Werror`; Python baseline rerun is + **159 passed, 41 skipped**; both repository `git diff --check` calls passed. + Ordinary native build retains only the observed duplicate-library linker + warning. Follow-up concurrent error-path FP restoration assertions and the + sinpi/cospi increment below are also covered by the final integrated rerun. + +## Further continuation: pi-scaled math accuracy + +- Replaced staged sinpi/cospi's large `pi*x` intermediate with exact modulo-two + reduction and reflection in pi units before libm evaluation. Integer and + half-integer anchors are exact, including large float32/float64 inputs; near + half-integer cospi avoids cancellation. Documented signed-zero anchors and + non-finite NaN behavior; frozen/full DSL is untouched. +- Added integer/half-integer/large/negative-zero/near-half/infinite input fixtures + for both float precisions. Concurrent shared-handle fixtures now also assert + FP environment restoration after checked-arithmetic failure. +- Final verification: **343/343 native tests passed**, targeted ASAN/UBSAN + interpreter/artifact **2/2 passed**, typed expression source warning check + **passed** with `-Wall -Wextra -Werror`. Python regression remains the latest + **159 passed / 41 skipped** run above. No genuine blocker occurred; beta gates + remain implementation work, not a scope reduction or public 1.0 certification. + +### Precise remaining work after these increments + +1. Finish static cardinality/placement conformance (including returned lanes and + reduction-valued nested flow), per-operation domain/tolerance/type matrix, and + platform-specific strict-FP certification. Extended internal evaluator has not + been certified as a complete artifact execution contract. +2. Connect descriptor-based numeric 1.0 artifact compilation/evaluation only after + validation, typed constants and immutable signature/cardinality are enforced; + finish actual schema 1.0 decoding and required-context/capability queries. +3. Add strings and ND using existing machinery with profile-aware semantics; + verify byte widths, capacities, coordinates and empty logical blocks. +4. Then complete Python bridge/logical scheduling/persistence/legacy policy and + platform/wheel/documentation gates. Keep published pin and public version gates. + +## Continuation: native schema/signatures, strings, ND, and staged Python bridge + +- `me_artifact_load` now dispatches schema/language 1.0 to profile-aware typed + interpreter compilation. Immutable signatures enforce all eleven numeric types, + exact width-checked signed/unsigned constants, IEEE bits, fixed-string widths, + Unicode32BE/byte hex constants, result cardinality and context rank. Unknown, + missing and unimplemented capabilities/operations/signatures explicitly reject. + Source tree scanning requires used numeric/control/reduction/string features. + Draft acceptance is not full certification; 0.1 validator/ABI remain frozen. +- Added internal uniform-input signature marker for strong captured constants; + constants are broadcast in call-local buffers without mutating shared handles. + Descriptor evaluation bypasses legacy tile execution/JIT; legacy eval rejects + 1.0 handles. Added native schema/capability/rank queries. +- Reused existing fixed-string validators, width inference, Unicode 15.0.0 case + tables and kernels through stack-local materialized operand adapters, not a new + parser/engine. Typed nodes handle string values, nested operations, predicates, + concat, lazy where and string-width fixed-point joins. Non-string numeric ops + reject string operands; explicit bool/string truth is defined by first code unit. + Active Unicode input scalars validate; masked padded inputs are not read. +- ND coordinates come only from explicit descriptor shape/origin/extent: checked + row-major logical `_flat_idx`, `_iN`, logical `_nN`, rank. Extent product equals + nitems; out-of-domain padded coordinates require invalid masks. Context buffers + and masks cannot overlap output. There is no implicit physical/global synthesis. +- End-to-end native artifact fixtures cover masked scalar/empty results, exact + uint64 max constants, missing capabilities/cardinality/rank, thirteen byte-string + operations (nested/where), Unicode expansion/truncation/constant endian encoding, + invalid Unicode, and padded 2D coordinates. Target ordinary native tests pass. +- ASAN found an adapter header copy over-read of zero-arity expression allocations; + fixed by copying only `offsetof(me_expr, parameters)` into owned stack headers. + Sanitized artifact executable now passes with `ASAN_OPTIONS=symbolize=0` (the + local symbolizer stalled when reporting the now-fixed finding). Interpreter + sanitizer target also passed. Final full rerun follows below. +- Started conditional Python native bridge: descriptor availability, authoritative + descriptor_info, and explicit evaluate_block (scalar shape(), fixed widths, + masks and logical ND context). Cython generation passed; new optional installed- + native tests never search a sibling checkout. No native pin/build replacement. + +### Exact next task at session limit + +1. Add returned-lane/reduction-local/string concurrent fixtures and finish finite + operation domain/type matrix certification (special domains, float32 result + types, literal/strong operand combinations, endian and width bounds). Dynamic + integral substr/split indices, replace bounds for typed constants/dynamic + strings, and explicit required-JIT rejection were completed subsequently. +2. Complete logical block-grid scheduling and partial reads in the lazy host. + `PortableKernel.from_json`, signature/cardinality/rank metadata, standalone + evaluate and explicit evaluate_block now route to native descriptors when + the installed ABI supports them, without Python reconstruction. Authoring 1.0, + lazy scheduling and persistence remain unchanged. Preserve pin independence; + actual compiled Python/native 1.0 execution tests currently require an opt-in + runtime and skip normally. Low-level Cython generation and bridge header compile + pass; host routing is tested with an explicit test double, not a language engine. +3. Complete all safe persistence routes and explicit legacy policy propagation; + then platform/wheel/docs gates. No genuine blocker or scope conflict occurred. + +- Subsequent native signature increment: integral source/captured/per-lane string + indices materialize into existing kernels with safe bounded clamping; + substring dynamic lengths retain subject width. Replace derives a checked + literal-aware/worst-case bound from capacities, rejecting widths over one MiB + and empty participating needles. Added negative index, UINT64_MAX length, + typed byte constants/replacement and runtime-empty-needle fixtures. Ordinary + and ASAN/UBSAN artifact executables passed. Added explicit `jit-required` + unsupported diagnostic; final integrated rerun follows. + +## Latest integrated verification and resume boundary + +### Current continuation: logical partitions and persistence (local native build) + +- Rebuilt the editable extension in conda `blosc2` using supported CMake + `FETCHCONTENT_SOURCE_DIR_MINIEXPR=/Users/faltet/blosc/miniexpr` and + `FETCHCONTENT_SOURCE_DIR_BLOSC2=.../build-portable-python/_deps/blosc2-src` + development overrides, `pip install -e . --no-build-isolation`. Published + FetchContent pin is unchanged. All three installed-native descriptor tests + now execute and pass; they do not skip in local verification. +- Added `src/blosc2/portable_lazy.py` native-only `PortableLazyArray`, constructed + via `PortableKernel.lazy`. Logical domain/partition tuples are independent of + operand storage grids. Scalar returns allocate the C-order partition grid; + elementwise returns allocate the original domain. Edge groups use actual valid + extents (no padded storage lanes synthesized). ND calls preserve original + logical shape/origin. Native partition lane limits validate at construction. +- Basic slices/integers/ellipsis/negative strides select intersecting original + groups, evaluate each whole group, and only then gather selected output lanes. + Partial scalar reads never repartition a reduction. Fancy indexing conservatively + evaluates the full original result before indexing. `compute`/`rechunk` preserve + partitions while choosing output storage chunks/blocks. Full compute streams + output storage chunks using selected original groups, avoiding full-result + NumPy allocation. Groups crossing output chunk boundaries may be evaluated + repeatedly but are never repartitioned. Partial/fancy reads return NumPy values. + Broadcast operands materialize against each group's extent while preserving + output-domain ND coordinates, with a partial-read regression fixture. +- Portable disk carriers, cframes, MessagePack structured objects, EmbedStore, + DictStore, TreeStore and nested arithmetic LazyExpr payloads roundtrip with + native artifact JSON and logical partitions, no originating function/source + reconstruction. Native artifact load precedes operand binding. Normal tests + never infer a sibling directory; local overrides are development-only. +- Removed new full/Python LazyUDF persistence and the old source fallback. + Shared recursive preflight runs before store overwrite/delete, and LazyUDF + save/frame rejects before touching a destination. Raw legacy UDF carriers + reject on NDArray save/frame too. Existing source-backed files still load only + under explicit effective `deserialize='full'`; policy propagates recursively + through structured operand refs and old LazyArray refs. No migration added. +- Safe arithmetic LazyExpr envelopes now have an AST whitelist (no calls, + attributes, subscriptions or ambient names) before host construction, with + safe policy propagated to every nested operand. Legacy nested DSL cannot hide + behind an arithmetic or portable carrier; explicit full remains required. +- CTable computed/generated legacy DSL metadata is policy-detected before + reconstruction; persistent add/save/frame/copy/physical-pack routes reject + full-DSL source descriptors before output writes. In-memory full DSL computation + remains supported. **Native portable CTable computed/generated transformer + descriptors are still unfinished**; do not claim table integration complete. +- Current new tests cover original-group reads, physical grid mismatch, partial + slices/rechunking, block-scalar grid shapes, ND/Unicode/mixed disk roundtrips, + all five container routes, native-only guard, preserved overwrite destinations, + effective nested legacy policy and CTable policy/preflight. Existing legacy + roundtrip tests now assert the new save rejection, while frozen legacy load + fixtures remain tested with explicit full. Removed 22 redundant external-corpus + Python parameter cases; exhaustive conversion coverage remains native. + +Changed Python inventory added this continuation: `src/blosc2/portable_lazy.py`, +`portable_kernel.py`, `b2objects.py`, `lazyexpr.py`, `schunk.py`, `core.py`, +`ndarray.py`, `msgpack_utils.py`, `embed_store.py`, `dict_store.py`, `tree_store.py`, +`ctable.py`; tests `test_portable_lazy.py`, `test_portable_policy.py`, +`test_b2objects.py`, `test_deserialization.py`, `test_dsl_portable.py`, and +`ctable/test_ctable_computed_cols.py`. Native edits from previous increments +remain preserved; no publishing, pin change or git state operations occurred. +User documentation updated in `doc/reference/portable_dsl.rst` for draft logical +partitions, partial reads, persistence routes and explicit legacy policy. + +Exact next implementation: portable CTable computed/generated descriptors and +row-domain/append/refresh semantics. Full DSL table sources now reject new +persistence but native portable table transformers are not connected yet. Then +finish authoring 1.0 records and the finite native conformance/platform gates. + +Latest verification for this continuation: + +- Combined relevant Python portable/persistence/CTable suite: **315 passed, + 19 skipped**. All actual 1.0 descriptor/lazy/ND/string runtime tests execute, + none skip. The 19 skips are 18 opt-in external 0.1 corpus fixtures and one + opt-in standalone native artifact runner; stock tests still do not discover + sibling sources. Portable-only coverage remains compact after replacing + redundant conversion/corpus parameter cases with orthogonal host boundaries: + **181 passed + 19 skipped = 200 portable Python cases** in the focused run. +- Full native CTest rerun passed (**343 tests**); targeted ASAN/UBSAN artifact and + interpreter rerun **2/2 passed** with symbolization disabled as documented. +- Ruff lint/format for all 18 affected Python source/test files passed; both repo + `git diff --check` calls passed. Local editable build completed successfully; + the three previously skipped installed-native descriptor fixtures passed. +- CTable computed/generated tests now **96 passed**, keeping full DSL local + computation while asserting new source persistence rejection. Frozen legacy + file load remains tested under explicit full. No external platform CI or wheel + matrix executed; do not claim beta readiness. No semantic blocker encountered. + +- Final native build and full CTest run: **343/343 passed**, including all new + schema/signature/string/ND/index/replacement fixtures. Build log retains only + the existing duplicate-library linker warning. +- Final targeted ASAN/UBSAN interpreter/artifact CTest run: **2/2 passed** with + `ASAN_OPTIONS=symbolize=0` to avoid the local symbolizer hang. The real adapter + over-read was fixed before this passing run; it is not being suppressed. +- `dsl_portable_expr.c` and `dsl_artifact.c` independently compiled with + `-Wall -Wextra -Werror`. Cython generation and bridge header syntax checks for + both descriptor-enabled and unavailable branches passed. This is not yet an + end-to-end compiled Python 1.0 runtime certification. +- Python relevant tests: **160 passed, 44 skipped**; three additional skips are + intentional installed-runtime 1.0 descriptor fixtures. The extra passing host + fixture checks native-authoritative metadata, byte-order/stride normalization + and explicit block routing without a source interpreter/test engine. +- Ruff lint/format and both repository diff whitespace checks passed. Source + byte-string literals now contextualize safely in where/output families, with + ASCII checks and bounded UTF32-to-byte materialization; native fixture passes. +- First resumed implementation task: connect `PortableKernel` standalone block + routing to lazy **logical partition scheduling** (immutable C-order block grid, + block-scalar grid result shape, full-group evaluation before partial selection, + explicit ND origin/extent and padding masks), then persistence/legacy policy. + Keep old installed runtime tests independent. In parallel with those dependency + milestones, finish the finite conformance/platform checklist above, not another + numerical approximation polishing loop. No genuine blocker encountered. + +## Files changed in this implementation session (current inventory) + +Python-Blosc2: + +- `plans/portable-DSL-1.0-progress.md`: durable handoff and verification. +- `src/blosc2/dsl_artifact_bridge.h`: version-guarded descriptor ABI/fallbacks. +- `src/blosc2/blosc2_ext.pyx`: low-level signature and block-evaluation bridge. +- `src/blosc2/portable_kernel.py`: authoritative descriptor metadata/standalone + block routing with explicit ND context, no Python source reconstruction. +- `tests/test_portable_descriptor.py`: optional installed-native 1.0 fixtures. +- `tests/test_portable_descriptor_host.py`: pin-independent host routing fixture. +- Previously authored scope/implementation plans and unrelated untracked work + remain preserved. No native pin, public authoring or persistence changes yet. + +Miniexpr: + +- `CMakeLists.txt`: add typed helper/expression sources and native integration tests. +- `src/dsl_semantic_profile.h`: per-program full/portable profile identities. +- `src/dsl_portable_expr.h`, `.c`, `dsl_portable_fp.h`: typed tree validation, + checked interpreter, libm/reductions/masks/string values and local FP policy. +- `src/dsl_compile.c`, `dsl_compile_internal.h`, `dsl_compile_support.c`: + profile compilation, fixed-point local joins/widths/cardinality, ownership. +- `src/dsl_eval.c`, `dsl_eval_internal.h`: masked statement integration, + initialization tracking, scalar/empty/descriptor/ND evaluation. +- `src/dsl_parser.c`, `dsl_parser.h`: profile token/parser integration. +- `src/functions.c`, `functions.h`, `src/miniexpr.c`, `miniexpr_internal.h`: + raw typed expression profile, operator identities, string-kernel adapters. +- `src/dsl_compare.c`: profile-aware integral comparison integration. +- `src/dsl_jit_runtime_internal.h`, `dsl_jit_runtime_host.c`, + `dsl_jit_runtime_nonhost.c`: staged interpreter/JIT guards and program metadata. +- `src/dsl_artifact.c`, `miniexpr_artifact.h`: draft schema/signatures/constants, + capabilities/context/cardinality queries and checked descriptor API. +- `tests/test_dsl_artifact.c`, `test_dsl_portable_interp.c`: end-to-end fixtures. +- `src/dsl_portable_types.h`, `src/dsl_portable_types.c`: internal operand + promotion and division dtypes; checked signed/unsigned integer arithmetic, + floor division/modulo, shifts, nonnegative powers, fixed-width bitwise + operations, float-to-integer and exact integer conversions/comparisons. +- `tests/test_dsl_portable_types.c`: explicit 121-pair expected promotion matrix. +- `tests/test_dsl_portable_checked.c`: exhaustive small-domain and extrema tests. +- `doc/dsl-spec/1.0.md`, `artifact-1.0.md`, `features-1.0.json`: native drafts; + not a claim of implemented 1.0 or a frozen specification. + +Helpers define left shift as checked multiplication, signed right shift as floor +division by a power of two, and nonnegative integer powers (`0**0=1`). Fractional +weak floating literals next to integers/Booleans remain floating rather than +being truncated. These are implementation-led details documented in the draft. diff --git a/plans/portable-DSL-1.0-scope-inventory.md b/plans/portable-DSL-1.0-scope-inventory.md new file mode 100644 index 000000000..4563f230f --- /dev/null +++ b/plans/portable-DSL-1.0-scope-inventory.md @@ -0,0 +1,245 @@ +# Portable DSL 1.0: agreed scope and specification work + +Status: feature-level scope agreed on 2026-10-07. Version 1.0 is the intended +first public compatibility contract; its detailed specification remains draft +until the acceptance gates below pass. Draft 1.0 is the sole public portable +format/default; the unreleased 0.1 path is retired with unsupported-version rejection. + +## Goal + +Persist DSL kernels only when their normalized source, typed bindings, and +execution requirements have a language-independent contract. Validate on save +and independently on load. Preserve convenient authoring syntax when it can be +lowered without retaining Python execution or live Python state. + +Interpreter-first: missing JIT support is not grounds for excluding a feature. +Select fallback before execution. A valid artifact can require capabilities +which a particular host lacks; that host must reject it explicitly. + +## Agreed inclusions + +- Real/integer/Boolean arithmetic, mixed numeric inputs, casts, predicates, + and an explicitly enumerated math-function set, including elementwise `where`. +- Existing branches, loops, local assignments, and loop control. +- ND index/shape symbols with explicit host-supplied logical context. +- Fixed-width Unicode strings/bytes and statically bounded string returns. +- Supported authoring rewrites and immutable typed scalar captures; static + column references become explicit named bindings. +- Limited block-local reductions: top-level reductions and supported + control-flow conditions, including Mandelbrot-style `all(...)` early exit. +- Portable kernel persistence with validation after normalization on save and + independent native validation on load. Unsupported kernels must fail saving; + no silent Python-specific persistence fallback for this path. +- Interpreter-first semantics; JIT is optional and may fall back before execution. + +## Inventory: prioritize semantic difficulty + +| Feature | Current implementation evidence | Portability work / difficulty | Agreed feature-level disposition | +| --- | --- | --- | --- | +| Block reductions inside kernels | Native DSL admits top-level reductions and reductions in conditions; rejects reductions inside control-flow bodies (`src/dsl_compile.c`). | **High.** Explicit block grouping, masking, scalar outputs/broadcast, accumulation order, empty blocks and padding. | Include limited reductions over the current evaluation block, including Mandelbrot `all` conditions. Exclude unsupported reductions inside control-flow bodies; no implicit full-array reduction. | +| External functions and closures | Native `me_variable` supports registered C functions/closures. Python authoring allows named calls syntactically; native compilation determines availability. | **Very high.** Function implementation identity/version, state, ABI, registration, and reproducibility cannot be represented by process-local pointers. | Exclude externally registered functions, callbacks, and opaque captured objects. Allow supported builtin functions and typed constant snapshots. | +| Complex arithmetic | Native complex types exist, but Windows rejects them because of complex ABI incompatibilities (`doc/data-types.md`, `doc/windows-limitations.md`). | **High.** Portable representation, branch cuts, non-finite behavior, promotion, and a functioning interpreter across target platforms. | Exclude complex inputs, intermediates, outputs, and constants for 1.0. This is a real reduction from the full native language. | +| Observable side effects | Native DSL implements `print`, restricted to uniform/scalar values. Arbitrary Python I/O is not native DSL. | **High for deterministic behavior.** Ordering and repetition depend on block scheduling, parallelism, partial evaluation, and repeated lazy evaluation. | Exclude `print` and external effects from portable kernels. Do not silently strip them. Host diagnostics remain execution policy. | +| Numeric promotion and conversions | Current DSL supports arithmetic, casts, mixed operands, predicates, and math more broadly than portable 0.1. | **High but essential.** Define intermediate/result types, literal typing, division, casts, predicate rounding, and a bounded supported function list. | Include ordinary real/integer/Boolean numerical use cases; interpreter-first. Do not promise all native numeric combinations automatically. | +| ND index/shape symbols | `_flat_idx`, `_i`, `_n`, `_ndim` already have native synthesis and Python constructor tests. | **Medium-high.** Explicit logical shape, evaluation region, origin/coordinates, slicing/view interpretation, broadcasting and padding rules. | Include a defined ND-context capability; do not expose physical chunk layout as logical semantics. Unsupported host contexts must reject. | +| Fixed-width strings and bytes | Native string locals/returns, width inference, comparisons, concat, transforms, substring and split operations; Python string DSL tests. Interpreter-only today. | **Medium-high.** Typed widths, endianness/code units, constants, first-NUL semantics, truncation, Unicode mapping version and whitespace tables. | Include a bounded fixed-width string/bytes capability. Existing operations are candidates, not all certified yet. | +| Variable-length string representation | Native `me_eval_varlen()` packs evaluated fixed-width values into Arrow-style offsets/data. Intermediates stay fixed-width. | **Medium for an adapter; high for a new language type.** Layout, capacity, offsets and ownership versus dynamically sized intermediates. | Defer a portable varlen ABI/type. Do not reject a fixed-width kernel merely because its host stores the result as varlen. | +| Authoring conveniences | `_NumpyAttrCallRewriter`, `_StringSyntaxRewriter`, `_RowSubscriptRewriter` in `dsl_kernel.py`. | **Low-medium.** Correct normalization, name resolution, evaluation order, typed captures and explicit column mappings. | Include supported normalization: `np.sin(x)`, string methods/membership/split sugar, static `row["col"]`. Dynamic DataFrame operations are not an existing native DSL feature. | +| Persisted graph and storage bindings | `b2objects.py` and `lazyexpr.py` preserve sources and operand references; reconstruction still uses `kernel_from_source`. CTable has DSL-backed column round trips. | **High integration scope.** Kernel portability does not define remote access, authentication, recursive graph execution, or storage identity. | Integrate portable kernel payloads and explicit bindings in the host container. Exclude arbitrary Python-object persistence; do not claim all host graphs/storage protocols are portable. | + +## Agreed exclusion list + +These are the kernel exclusions for the first public release: + +1. **Unsupported reductions inside control-flow bodies and implicit global + collectives.** Permit top-level reductions and supported reduction conditions, + including an `if all(...)` condition within a loop. This is distinct from a + reduction assignment/return inside the body. No implicit reinterpretation of + block reductions as full-array reductions. Classify operations by semantics: + elementwise `fmin`/`fmax` are not block reductions. +2. **Externally registered functions, closures, and runtime object dependencies.** + No function pointers, Python callbacks, opaque object captures, or implicit + module/global state. Supported scalar captures become immutable typed values; + arrays/columns become explicit host bindings rather than hidden captures. +3. **Complex-valued computation.** Defer until there is a portable cross-platform + interpreter contract, independently of JIT availability. +4. **Observable kernel side effects**, including native `print`. Host tracing is + not a persisted kernel effect. +5. **Non-strict FP semantic modes** (`fast`/`contract`) for the initial contract. + Strict arithmetic and explicitly documented math-function accuracy are the + baseline; this does not imply bitwise-identical transcendental results across + all platforms. Persist semantic requirements, not compiler paths or caches. +6. **Dynamically sized intermediate/output types and arbitrary memory access.** + Retain statically bounded string widths and explicit output cardinality + (elementwise or supported block-scalar reduction). + General gathers/scatters, mutation of inputs, arbitrary array subscripting, + and dynamically selected columns are not new 1.0 features. Several of these + are already outside today's DSL; distinguish them from lost capabilities. + +Separate exclusions from the *artifact/host scope*: generated machine code, +Python source execution on import, pickled runtime objects, external function +registration protocols, and universal graph/remote-storage execution. Saving a +portable kernel in a LazyArray does not itself make every referenced data source +available in C or another language. + +## Block-local reduction contract to specify + +The reduction group is the **current evaluation block**, not the full logical +array. Partition-dependent results are permitted; reproducing those results +requires preserving the evaluation partition. The persistence/host integration +must record or unambiguously reconstruct that partition, not silently retile it. +Logical ND coordinates and physical evaluation grouping remain distinct concepts. + +The mandatory motivating case is `if all(escape_iter != limit): break` in the +Mandelbrot loop. It is a block-level early-exit optimization, not an array-wide +synchronization primitive. Conformance must test different partitions and show +both this use case and a deliberately partition-dependent reduction result. + +The approved policies below settle valid-element/active-mask participation, +empty identities, scalar broadcast, partition preservation, and ordered sums. +Before freezing, complete output-cardinality and per-operation details. +Masked/nested conditions that current implementations cannot +define consistently must be identified explicitly, not accidentally admitted. + +## Hard numeric domains: not syntactic exclusions + +Overflow, non-finite/out-of-range integer casts, division by zero, and invalid +shift counts depend on runtime values. They cannot generally be detected by a +save-time validator. Do not claim that successful persistence proves their +absence. The agreed rules below require runtime errors for the specified invalid +integer operations and conversions, and IEEE-style floating behavior. Implement +those checks rather than relying on caller preconditions for these cases. + +Likewise, unsigned integers, narrower widths, and each math builtin need an +explicit type/function matrix. They are open inventory decisions, not covered +by a blanket claim of full miniexpr compatibility. Real arithmetic remains a +major specification task even with limited reductions and no complex numbers. + +## What should not be excluded merely to save JIT work + +- Integer-input division, supported mixed numeric inputs, nested casts and + arithmetic predicates, once their interpreter semantics are defined. +- An agreed set of math builtins and elementwise `where`. +- ND symbols supplied through a defined logical context. +- Fixed-width Unicode/bytes kernels, including bounded string returns. +- Static column access and supported NumPy/string authoring sugar after lowering. + +For example, the existing row rewrite accepts a single row parameter used only +as `row[]` and records parameter-to-column mappings. It is not a +general translation of arbitrary `df[expr]` indexing. + +## Evidence and documentation gaps + +- `../miniexpr/doc/dsl-syntax.md`: statements, ND context, string locals/returns, + registered functions, reductions and print. +- `../miniexpr/doc/strings.md`: UCS4 versus bytes, NUL termination, width bounds, + Unicode truncation, and the existing varlen output adapter. +- `../miniexpr/src/dsl_compile.c`: actual reduction restrictions; existing + unsupported combinations are not an additional portable-only limitation. +- `src/blosc2/dsl_kernel.py`: authoring validator and normalization boundaries. +- `tests/ndarray/test_dsl_kernels.py`, `tests/ndarray/test_string_output.py`, and + `tests/ctable/test_ctable_dsl_columns.py`: ND, string, and persistence use cases. +- `src/blosc2/b2objects.py`, `src/blosc2/lazyexpr.py`: existing persistence paths. + +Some references disagree: Python's DSL page still describes reduction-in-body +behavior as silently incorrect, while current native compilation rejects it; +the strings guide's scalar-condition wording also needs reconciliation with +masked string DSL behavior. Resolve against implementation/tests before freezing +normative text. This inventory is a source review, not new feature certification. + +## Agreed semantic and compatibility decisions + +The following choices were approved after the feature-level scope discussion. +They govern portable 1.0; they are not claims about existing full-DSL behavior. + +### Numeric computation + +- Operand types and explicit casts determine intermediate computation; output + conversion happens last. An output dtype must not change the calculation. + In particular, float32 `(x + 1.0) - x` computes in float32 even when the output + is float64. Apply the new rules through the versioned portable path without + silently redefining ordinary full-DSL execution. +- Literals adopt the operand type where representable; otherwise use explicit + documented promotion rules or reject unrepresentable literals. Mixed typed + operands follow one enumerated promotion table. +- Integer `/` produces float64; homogeneous float32 division stays float32. +- `int(expr)` evaluates its argument first, then truncates toward zero. +- Boolean arithmetic uses numeric zero/one. Explicit `bool()` and Boolean output + perform truth conversion. +- Integer overflow, out-of-range integer narrowing, non-finite/out-of-range + float-to-integer conversion, integer floor-division/remainder by zero, and + invalid shift counts are runtime errors. +- Floating arithmetic retains IEEE-style NaNs, infinities and signed zero, + including floating division by zero. Enumerate math-function domain behavior + and accuracy guarantees separately; strict does not imply universally + bitwise-identical transcendental results. +- JIT must implement the same checks and semantics or fall back before execution. + +### Block reductions + +- Reductions consume valid, currently active lanes of the evaluation block. + Top-level reductions see all valid block elements. Supported nested conditions + see their participating lanes; exclude padding and lanes that have returned + or left the relevant loop. +- `any(empty)` is false, `all(empty)` is true, and `sum(empty)` is zero. + Empty `min`/`max` is a runtime error. Specify other admitted reductions in the + operation matrix before freezing. +- A reduced scalar controls or broadcasts only to participating lanes. +- Persist the logical evaluation partition independently of physical storage. + Rechunking must not silently change reduction groups. Partial reads evaluate + the original groups needed for the slice, then select requested results. + Explicit repartitioning may change results and must be described as such. +- Specify accumulation dtype and logical lane order for floating sums; initially + exclude reassociation into a different reduction tree. Integer accumulation + obeys checked-overflow rules. Relaxed reductions are a later capability. + +### Persistence and legacy formats + +- New DSL-kernel saves normalize authoring syntax/captures, validate portable + 1.0, and persist the artifact, bindings and required execution context. +- Unsupported kernels fail saving with actionable diagnostics; there is no + silent fallback to Python-specific persistence. +- Detect legacy persisted kernel formats and require explicit opt-in before + using them, through the applicable deserialization policy. Never execute + legacy Python reconstruction merely to detect the format. +- **No migration tooling or automatic conversion path is in scope.** Users who + want a portable 1.0 artifact must follow the guidelines, manually adapt their + kernels/bindings, and save through the validated 1.0 path. Legacy opt-in is + permission to use the old representation, not a claim of portable compliance. +- Publication guidance must document supported operations, the numeric behavior + change, reduction grouping, and legacy opt-in/manual adaptation. + +## Remaining specification drafting + +Implementation-led drafting can cover schema fields, capability negotiation, +typed encodings, explicit column bindings, existing ND coordinate conventions, +string width/Unicode rules, diagnostics, and compact conformance scenarios. +Use verified interpreter behavior as evidence, not inconsistent documentation or +an accidental backend behavior as the normative contract. + +The policy choices above are settled. Draft the exact promotion table, literal +representability rules, function/type matrix, reduction NaN behavior and +accumulation dtypes, mask transitions, partition encoding, and legacy format +discriminators/opt-in API against them. Surface contradictions or additional +compatibility consequences for review rather than changing approved policy. + +Draft an explicit dtype/function matrix based on current use cases; surface any +proposed omission of an existing useful operation for review. Pin string mapping +behavior and document truncation rather than silently redesigning it. Do not +promise bitwise-identical transcendental results without an implementation that +can deliver them. + +## Bounded acceptance gates + +1. Draft and review the detailed 1.0 semantics, type/function matrix, capability + requirements, and detailed rules implementing the agreed decisions above. +2. Implement native validation and interpreter conformance for those contracts; + reject unsupported kernels with actionable diagnostics. +3. Integrate portable payloads in kernel persistence, preserving typed bindings, + ND context and reduction partition requirements. Verify standalone C loading + without Python reconstruction, plus host save/load round trips. +4. Certify supported platforms. JIT must preserve the contract or select fallback + before execution; additional acceleration is not a release gate. + +Keep tests orthogonal and retain native ownership of exhaustive language fixtures. +Scope changes require explicit agreement; do not restart a whole-language JIT audit. diff --git a/src/blosc2/b2objects.py b/src/blosc2/b2objects.py index fc4c2cac7..a447c1051 100644 --- a/src/blosc2/b2objects.py +++ b/src/blosc2/b2objects.py @@ -7,16 +7,16 @@ from __future__ import annotations -import inspect +import ast import pathlib -import textwrap -from dataclasses import asdict from typing import Any import numpy as np import blosc2 -from blosc2.dsl_kernel import DSLKernel, kernel_from_source +from blosc2.deserialization import DeserializeMode, get_deserialize, normalize_deserialize +from blosc2.dsl_kernel import kernel_from_source +from blosc2.exceptions import UnsafeDeserializationError _B2OBJECT_META_KEY = "b2o" _B2OBJECT_VERSION = 1 @@ -24,6 +24,23 @@ _B2OBJECT_USER_VLMETA_KEY = "_b2o_user_vlmeta" +def preflight_persistence(obj): + """Validate executable descriptors recursively before a destination is touched.""" + if isinstance(obj, blosc2.LazyArray): + encode_b2object_payload(obj) + from blosc2.msgpack_utils import msgpack_packb + + msgpack_packb(obj._get_user_vlmeta()) + elif isinstance(obj, blosc2.CTable): + obj._preflight_portable_persistence() + else: + schunk = getattr(obj, "schunk", obj if isinstance(obj, blosc2.SChunk) else None) + if schunk is not None: + marker = schunk.meta.get("b2o", {}) + if marker.get("kind") == "lazyudf" or schunk.meta.get("LazyArray") == 1: + raise TypeError("Persisting legacy DSL/Python UDF metadata is unsupported") + + def make_b2object_carrier( kind: str, shape, @@ -55,10 +72,14 @@ def read_b2object_user_vlmeta(obj) -> dict[str, Any]: def encode_operand_reference(obj): + from blosc2.portable_lazy import PortableLazyArray + + if isinstance(obj, PortableLazyArray | blosc2.LazyExpr | blosc2.LazyUDF): + return encode_b2object_payload(obj) return blosc2.Ref.from_object(obj).to_dict() -def decode_operand_reference(payload, *, base_path=None): +def decode_operand_reference(payload, *, base_path=None, deserialize="safe"): if ( payload.get("kind") in {"urlpath", "dictstore_key"} and base_path is not None @@ -67,10 +88,21 @@ def decode_operand_reference(payload, *, base_path=None): payload = dict(payload) payload["urlpath"] = (base_path / payload["urlpath"]).as_posix() ref = blosc2.Ref.from_dict(payload) - return ref.open(deserialize="full") + return ref.open(deserialize=deserialize) def encode_b2object_payload(obj) -> dict[str, Any] | None: + from blosc2.portable_lazy import PortableLazyArray + + if isinstance(obj, PortableLazyArray): + return { + "kind": "portable", + "version": _B2OBJECT_VERSION, + "artifact": obj.kernel.to_json(), + "shape": list(obj._domain), + "partitions": list(obj.partitions), + "operands": {name: encode_operand_reference(value) for name, value in obj.inputs.items()}, + } if isinstance(obj, blosc2.C2Array): return blosc2.Ref.c2array_ref(obj.path, obj.urlbase).to_dict() if isinstance(obj, blosc2.LazyExpr): @@ -83,66 +115,75 @@ def encode_b2object_payload(obj) -> dict[str, Any] | None: "operands": {key: encode_operand_reference(value) for key, value in operands.items()}, } if isinstance(obj, blosc2.LazyUDF): - if not isinstance(obj.func, DSLKernel): - raise TypeError("Structured Blosc2 msgpack payload only supports LazyUDF backed by DSLKernel") - udf_func = obj.func.func - udf_name = getattr(udf_func, "__name__", obj.func.__name__) - try: - udf_source = textwrap.dedent(inspect.getsource(udf_func)).lstrip() - except Exception: - udf_source = obj.func.dsl_source - if udf_source is None: - raise ValueError("Structured LazyUDF msgpack payload requires recoverable DSL kernel source") - kwargs = {} - for key, value in obj.kwargs.items(): - if key in {"dtype", "shape"}: - continue - if isinstance(value, blosc2.CParams | blosc2.DParams): - kwargs[key] = asdict(value) - else: - kwargs[key] = value - return { - "kind": "lazyudf", - "version": _B2OBJECT_VERSION, - "function_kind": "dsl", - "dsl_version": _B2OBJECT_DSL_VERSION, - "name": udf_name, - "udf_source": udf_source, - "dtype": np.dtype(obj.dtype).str, - "shape": list(obj.shape), - "operands": {f"o{i}": encode_operand_reference(value) for i, value in enumerate(obj.inputs)}, - "kwargs": kwargs, - } + from blosc2.portable_lazy import portable_from_lazyudf + + return encode_b2object_payload(portable_from_lazyudf(obj)) return None -def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carrier=None): +def decode_b2object_payload(payload: dict[str, Any], *, carrier_path=None, carrier=None, deserialize="safe"): + deserialize = normalize_deserialize(deserialize) kind = payload.get("kind") version = payload.get("version") if version != _B2OBJECT_VERSION: raise ValueError(f"Unsupported persisted Blosc2 object version: {version!r}") if kind == "c2array": ref = blosc2.Ref.from_dict(payload) - return ref.open(deserialize="full") + return ref.open(deserialize=deserialize) + if kind == "portable": + kernel = blosc2.PortableKernel.from_json(payload["artifact"]) + operands, missing = decode_operand_mapping( + payload["operands"], base_path=carrier_path, deserialize=deserialize + ) + if missing: + raise FileNotFoundError(f"Missing portable operands: {missing}") + return kernel.lazy(operands, shape=payload["shape"], partitions=payload["partitions"]) if kind == "remote_array": if carrier is None: raise ValueError("A persisted RemoteArray requires its B2ND carrier") return blosc2.RemoteArray._from_payload(payload, carrier) if kind == "lazyexpr": - return decode_structured_lazyexpr(payload, carrier_path=carrier_path) + return decode_structured_lazyexpr(payload, carrier_path=carrier_path, deserialize=deserialize) if kind == "lazyudf": - return decode_structured_lazyudf(payload, carrier_path=carrier_path) + if deserialize is not DeserializeMode.FULL: + raise UnsafeDeserializationError("legacy DSL LazyUDF") + return decode_structured_lazyudf(payload, carrier_path=carrier_path, deserialize=deserialize) raise ValueError(f"Unsupported persisted Blosc2 object kind: {kind!r}") -def decode_structured_lazyexpr(payload, *, carrier_path=None): +def decode_structured_lazyexpr(payload, *, carrier_path=None, deserialize="safe"): expression = payload.get("expression") if not isinstance(expression, str): raise TypeError("Structured LazyExpr payload requires a string 'expression'") operands_payload = payload.get("operands") if not isinstance(operands_payload, dict): raise TypeError("Structured LazyExpr payload requires a mapping 'operands'") - operands, missing_ops = decode_operand_mapping(operands_payload, base_path=carrier_path) + from blosc2.lazyexpr import validate_expr + + validate_expr(expression) + if normalize_deserialize(deserialize) is DeserializeMode.SAFE: + allowed = ( + ast.Expression, + ast.BinOp, + ast.UnaryOp, + ast.BoolOp, + ast.Compare, + ast.Name, + ast.Load, + ast.Constant, + ast.operator, + ast.unaryop, + ast.boolop, + ast.cmpop, + ) + if any( + not isinstance(node, allowed) or (isinstance(node, ast.Name) and node.id not in operands_payload) + for node in ast.walk(ast.parse(expression, mode="eval")) + ): + raise UnsafeDeserializationError("non-arithmetic lazy expression") + operands, missing_ops = decode_operand_mapping( + operands_payload, base_path=carrier_path, deserialize=deserialize + ) if missing_ops: exc = blosc2.exceptions.MissingOperands(expression, missing_ops) exc.expr = expression @@ -151,12 +192,17 @@ def decode_structured_lazyexpr(payload, *, carrier_path=None): return blosc2.lazyexpr(expression, operands=operands) -def decode_operand_mapping(operands_payload, *, base_path=None): +def decode_operand_mapping(operands_payload, *, base_path=None, deserialize="safe"): operands = {} missing_ops = {} for key, value in operands_payload.items(): try: - operands[key] = decode_operand_reference(value, base_path=base_path) + if value.get("kind") in {"portable", "lazyexpr", "lazyudf"}: + operands[key] = decode_b2object_payload( + value, carrier_path=base_path, deserialize=deserialize + ) + else: + operands[key] = decode_operand_reference(value, base_path=base_path, deserialize=deserialize) except FileNotFoundError: ref = blosc2.Ref.from_dict(value) if ref.kind in {"urlpath", "dictstore_key"}: @@ -166,7 +212,9 @@ def decode_operand_mapping(operands_payload, *, base_path=None): return operands, missing_ops -def decode_structured_lazyudf(payload, *, carrier_path=None): +def decode_structured_lazyudf(payload, *, carrier_path=None, deserialize="safe"): + if normalize_deserialize(deserialize) is not DeserializeMode.FULL: + raise UnsafeDeserializationError("legacy DSL LazyUDF") function_kind = payload.get("function_kind") if function_kind != "dsl": raise ValueError(f"Unsupported structured LazyUDF function kind: {function_kind!r}") @@ -194,15 +242,19 @@ def decode_structured_lazyudf(payload, *, carrier_path=None): func = kernel_from_source(udf_source, name) ordered_operands_payload = {f"o{n}": operands_payload[f"o{n}"] for n in range(len(operands_payload))} - operands, missing_ops = decode_operand_mapping(ordered_operands_payload, base_path=carrier_path) + operands, missing_ops = decode_operand_mapping( + ordered_operands_payload, base_path=carrier_path, deserialize=deserialize + ) if missing_ops: exc = blosc2.exceptions.MissingOperands(name, missing_ops) exc.expr = name exc.missing_ops = missing_ops raise exc - return blosc2.lazyudf( + result = blosc2.lazyudf( func, tuple(operands.values()), dtype=np.dtype(dtype), shape=tuple(shape_payload), **kwargs ) + result._legacy_source_recipe = True + return result def read_b2object_marker(obj) -> dict[str, Any] | None: @@ -231,8 +283,13 @@ def open_b2object(obj): schunk = getattr(obj, "schunk", obj) if getattr(schunk, "urlpath", None) is not None: carrier_path = pathlib.Path(schunk.urlpath).parent - opened = decode_b2object_payload(payload, carrier_path=carrier_path, carrier=obj) - if isinstance(opened, blosc2.LazyExpr | blosc2.LazyUDF): + opened = decode_b2object_payload( + payload, carrier_path=carrier_path, carrier=obj, deserialize=get_deserialize(obj) + ) + from blosc2.deserialization import set_deserialize + + set_deserialize(opened, get_deserialize(obj)) + if isinstance(opened, blosc2.LazyArray): opened.array = obj opened.schunk = schunk opened._set_user_vlmeta(read_b2object_user_vlmeta(obj), sync=False) diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index d7cfbeae1..fd17e3245 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -965,31 +965,45 @@ cdef extern from *: """ #include "miniexpr.h" #include + #include static int b2_validate_portable_dsl(const char *source, const char *version, const me_variable *inputs, int ninputs, me_dtype output_dtype, + size_t output_itemsize, int ndim, int cardinality, int *line, int *column, char *message, size_t capacity) { - #ifdef ME_PORTABLE_DSL_VERSION + #ifdef ME_PORTABLE_DSL_VALIDATION_DESCRIPTOR_VERSION + if (strcmp(ME_PORTABLE_DSL_VERSION, "1.0")) { + *line = *column = 0; + snprintf(message, capacity, "installed miniexpr lacks draft 1.0 source validation"); + return -100; + } me_portable_error error = {0}; - int rc = me_validate_portable_dsl(source, version, inputs, ninputs, - output_dtype, &error); + int rc; + me_portable_validation_descriptor descriptor = { + .struct_size = sizeof(descriptor), + .version = ME_PORTABLE_DSL_VALIDATION_DESCRIPTOR_VERSION, + .output_itemsize = output_itemsize, .ndim = ndim, + .cardinality = (me_portable_cardinality)cardinality}; + rc = me_validate_portable_dsl_ex(source, version, inputs, ninputs, + output_dtype, &descriptor, &error); *line = error.line; *column = error.column; snprintf(message, capacity, "%s", error.message); return rc; #else *line = *column = 0; - snprintf(message, capacity, "miniexpr was built without portable DSL validation support"); + snprintf(message, capacity, "miniexpr was built without draft 1.0 source validation support"); return -100; #endif } """ int b2_validate_portable_dsl(const char *source, const char *version, const me_variable *inputs, int ninputs, me_dtype output_dtype, + size_t output_itemsize, int ndim, int cardinality, int *line, int *column, char *message, size_t capacity) -def validate_portable_dsl_source(source, input_dtypes, output_dtype, version): - """Native-only portable 0.1 validation; no arrays, JIT, or Python execution.""" +def validate_portable_dsl_source(source, input_dtypes, output_dtype, version, int ndim=0, int cardinality=0): + """Native-only versioned source validation; no arrays, JIT or Python execution.""" cdef Py_ssize_t n = len(input_dtypes) if n > 128: raise ValueError("Too many portable DSL inputs (maximum 128)") @@ -1000,6 +1014,7 @@ def validate_portable_dsl_source(source, input_dtypes, output_dtype, version): cdef bytes name_bytes cdef np.dtype dtype cdef me_dtype result_dtype + cdef size_t output_itemsize = 0 cdef int rc, line = 0, column = 0 cdef char message[256] cdef Py_ssize_t i @@ -1016,13 +1031,18 @@ def validate_portable_dsl_source(source, input_dtypes, output_dtype, version): variables[i].dtype = _me_dtype_from_numpy_dtype(dtype) except TypeError: variables[i].dtype = ME_AUTO + if version == "1.0" and variables[i].dtype in (ME_STRING, ME_BYTES): + variables[i].itemsize = dtype.itemsize dtype = np.dtype(output_dtype) try: result_dtype = _me_dtype_from_numpy_dtype(dtype) except TypeError: result_dtype = ME_AUTO + if version == "1.0" and result_dtype in (ME_STRING, ME_BYTES): + output_itemsize = dtype.itemsize rc = b2_validate_portable_dsl(source_bytes, version_bytes, variables, n, - result_dtype, &line, &column, message, sizeof(message)) + result_dtype, output_itemsize, ndim, cardinality, + &line, &column, message, sizeof(message)) statuses = { 0: "success", -1: "unsupported_version", -2: "invalid_signature", -3: "invalid_source", -4: "unsupported_feature", -5: "out_of_memory", @@ -1058,12 +1078,40 @@ cdef extern from "dsl_artifact_bridge.h": me_dtype b2_artifact_dtype(const void *, int) me_dtype b2_artifact_output(const void *) int b2_artifact_jit(const void *) + ctypedef struct b2_artifact_buffer: + const char *name + me_dtype dtype + size_t itemsize + const void *data + size_t capacity + ctypedef struct b2_artifact_descriptor: + size_t struct_size + unsigned int version + size_t nitems + size_t output_capacity + const uint8_t *valid_mask + size_t valid_mask_capacity + int ndim + const int64_t *logical_shape + const int64_t *block_origin + const int64_t *block_extent + int b2_artifact_descriptor_available() + int b2_artifact_cardinality(const void *) + size_t b2_artifact_width(const void *, int) + int b2_artifact_rank(const void *) + const char *b2_artifact_version(const void *) + int b2_artifact_eval_ex(const void *, const b2_artifact_buffer *, int, void *, + const b2_artifact_descriptor *, b2_artifact_error *) nogil def portable_artifact_available(): return bool(b2_artifact_available()) +def portable_descriptor_available(): + return bool(b2_artifact_descriptor_available()) + + cdef void raise_artifact_error(int rc, b2_artifact_error *error) except *: from blosc2.portable_kernel import PortableArtifactError @@ -1109,6 +1157,104 @@ cdef class PortableArtifactHandle: "jit": bool(b2_artifact_jit(self._handle)), } + cdef object signature_dtype(self, int index): + cdef me_dtype dtype = b2_artifact_output(self._handle) if index < 0 else b2_artifact_dtype(self._handle, index) + cdef size_t width = b2_artifact_width(self._handle, index) + if dtype == ME_STRING: + return np.dtype(f"U{width // 4}") + if dtype == ME_BYTES: + return np.dtype(f"S{width}") + return _numpy_dtype_from_me_dtype(dtype) + + def descriptor_info(self): + """Authoritative draft native signature; unavailable on older pinned builds.""" + if not b2_artifact_descriptor_available(): + raise NotImplementedError("Build with draft miniexpr descriptor support") + return { + "schema_version": (b2_artifact_version(self._handle)).decode("ascii"), + "inputs": {(b2_artifact_name(self._handle, i)).decode("utf-8"): + self.signature_dtype(i) for i in range(b2_artifact_ninputs(self._handle))}, + "output_dtype": self.signature_dtype(-1), + "cardinality": "block_scalar" if b2_artifact_cardinality(self._handle) == 1 else "elementwise", + "ndim": b2_artifact_rank(self._handle), + } + + def evaluate_block(self, inputs, block_shape, *, logical_shape=None, block_origin=None, valid_mask=None): + """Evaluate one explicit logical block; never infer ND coordinates from a physical tile.""" + from math import prod + + if not b2_artifact_descriptor_available(): + raise NotImplementedError("Build with draft miniexpr descriptor support") + cdef Py_ssize_t n = len(inputs), i + if n > 128: + raise ValueError("Too many portable artifact inputs") + block_shape = tuple(block_shape) + if any(type(size) is not int or size < 0 for size in block_shape): + raise ValueError("Block shape must contain nonnegative Python integers") + count = prod(block_shape) + if count > 2147483647: + raise ValueError("Native block exceeds the supported lane count") + cdef b2_artifact_buffer *bindings = NULL + cdef b2_artifact_descriptor descriptor + cdef b2_artifact_error error + cdef np.ndarray array, output, mask, shape_array, origin_array, extent_array + cdef list names = [key.encode("utf-8") for key in inputs] + cdef list arrays = list(inputs.values()) + cdef bytes name + cdef int rc + memset(&descriptor, 0, sizeof(descriptor)) + descriptor.struct_size = sizeof(descriptor) + descriptor.version = 1 + descriptor.nitems = count + descriptor.ndim = b2_artifact_rank(self._handle) + if descriptor.ndim: + if logical_shape is None or block_origin is None or len(block_shape) != descriptor.ndim: + raise ValueError("Explicit logical shape, block origin and matching block rank are required") + shape_array = np.ascontiguousarray(logical_shape, dtype=np.int64) + origin_array = np.ascontiguousarray(block_origin, dtype=np.int64) + extent_array = np.ascontiguousarray(block_shape, dtype=np.int64) + if shape_array.ndim != 1 or origin_array.ndim != 1 or shape_array.shape[0] != descriptor.ndim or origin_array.shape[0] != descriptor.ndim: + raise ValueError("Logical context rank mismatch") + descriptor.logical_shape = np.PyArray_DATA(shape_array) + descriptor.block_origin = np.PyArray_DATA(origin_array) + descriptor.block_extent = np.PyArray_DATA(extent_array) + elif logical_shape is not None or block_origin is not None: + raise ValueError("This artifact does not declare ND context") + if valid_mask is not None: + mask = np.asarray(valid_mask) + if mask.dtype not in (np.dtype(np.bool_), np.dtype(np.uint8)) or mask.size != count: + raise ValueError("Valid mask must have one Boolean/uint8 slot per block lane") + mask = np.ascontiguousarray(mask, dtype=np.uint8) + descriptor.valid_mask = np.PyArray_DATA(mask) + descriptor.valid_mask_capacity = mask.nbytes + output = np.empty(() if b2_artifact_cardinality(self._handle) == 1 else block_shape, + dtype=self.signature_dtype(-1)) + descriptor.output_capacity = output.nbytes + if n: + bindings = calloc(n, sizeof(b2_artifact_buffer)) + if bindings == NULL: + raise MemoryError() + try: + for i in range(n): + array = arrays[i] + if not array.flags.c_contiguous or not array.flags.aligned or not array.dtype.isnative or array.size != count: + raise ValueError("Block inputs must be aligned, contiguous, host-endian and match the block extent") + name = names[i] + if b'\x00' in name: + raise ValueError("Input names must not contain NUL") + bindings[i].name = name + bindings[i].dtype = _me_dtype_from_numpy_dtype(array.dtype) + bindings[i].itemsize = array.dtype.itemsize + bindings[i].data = np.PyArray_DATA(array) + bindings[i].capacity = array.nbytes + with nogil: + rc = b2_artifact_eval_ex(self._handle, bindings, n, np.PyArray_DATA(output), &descriptor, &error) + if rc: + raise_artifact_error(rc, &error) + return output + finally: + free(bindings) + def evaluate(self, inputs, shape): cdef Py_ssize_t n = len(inputs) if n > 128: diff --git a/src/blosc2/core.py b/src/blosc2/core.py index b2ebfb75e..a3f250197 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -2392,7 +2392,7 @@ def from_cframe( if "b2o" in schunk.meta: marker = schunk.meta["b2o"] kind = marker.get("kind", "b2o") if isinstance(marker, dict) else "b2o" - if deserialize is DeserializeMode.SAFE: + if deserialize is DeserializeMode.SAFE and kind not in {"portable", "lazyexpr"}: raise UnsafeDeserializationError(str(kind)) return blosc2.open_b2object(ndarray_from_cframe(cframe, copy=copy, deserialize=deserialize)) if "b2nd" in schunk.meta: diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 64cdb8032..5ccb0d166 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -7007,6 +7007,7 @@ def to_b2z( """ if not str(urlpath).endswith(".b2z"): raise ValueError("urlpath must have a .b2z extension") + self._preflight_portable_persistence() if preserve_sources and self.base is not None: raise ValueError("preserve_sources requires an unfiltered root table") @@ -7075,6 +7076,7 @@ def to_b2d( """ urlpath = os.fspath(urlpath) storage = getattr(self, "_storage", None) + self._preflight_portable_persistence() can_physical_unpack = ( not compact and self.base is None @@ -7122,6 +7124,7 @@ def to_cframe(self, *, preserve_sources: bool = False) -> bytes: :func:`blosc2.ctable_from_cframe` """ # Materialize live rows for views/slices; for base tables use the live + self._preflight_portable_persistence() # columns directly. copy() is the canonical materialization path. if self.base is not None: src = self.copy(compact=True) @@ -7421,6 +7424,7 @@ def save(self, urlpath: str, *, overwrite: bool = False, preserve_sources: bool """ if preserve_sources and self.base is not None: raise ValueError("preserve_sources requires an unfiltered root table") + self._preflight_portable_persistence() if self.base is not None: materialized = self.copy(compact=True) materialized.save(urlpath, overwrite=overwrite) @@ -12164,6 +12168,8 @@ def _readable_computed_expr(cc: dict) -> str: ``kernel(dep0, dep1)`` call label is returned. """ col_deps = cc["col_deps"] + if cc.get("kind") == "portable": + return f"portable[{cc['row_domain']}]({', '.join(col_deps)})" if cc.get("kind") == "dsl": kernel = cc.get("kernel") kname = getattr(kernel, "__name__", "dsl_kernel") @@ -12258,7 +12264,20 @@ def _schema_dict_with_computed(self) -> dict: if self._computed_cols: computed = [] for name, cc in self._computed_cols.items(): - if cc.get("kind") == "dsl": + if cc.get("kind") == "portable": + computed.append( + { + "name": name, + "kind": "portable", + "artifact": cc["artifact"], + "bindings": cc["bindings"], + "row_domain": cc["row_domain"], + "row_shape": list(cc["row_shape"]), + "col_deps": cc["col_deps"], + "dtype": str(cc["dtype"]), + } + ) + elif cc.get("kind") == "dsl": entry = { "name": name, "kind": "dsl", @@ -12294,6 +12313,10 @@ def _schema_dict_with_computed(self) -> dict: } if "transformer" in meta: entry["transformer"] = meta["transformer"] + if meta.get("transformer_kind") == "portable": + entry.update( + {key: meta[key] for key in ("artifact", "bindings", "row_domain", "row_shape")} + ) if meta.get("dsl_source") is not None: entry["dsl_source"] = meta["dsl_source"] if meta.get("jit_backend") is not None: @@ -12488,11 +12511,14 @@ def _load_computed_cols_from_schema(self, schema_dict: dict) -> None: Called from ``__init__``, ``open``, and ``load`` after all stored columns have been opened into ``self._cols``. """ + self._check_legacy_dsl_policy(schema_dict) for cc_meta in schema_dict.get("computed_columns", []): name = cc_meta["name"] col_deps = cc_meta["col_deps"] dtype = np.dtype(cc_meta["dtype"]) - if cc_meta.get("kind") == "dsl": + if cc_meta.get("kind") == "portable": + self._computed_cols[name] = self._validate_portable_table_metadata(cc_meta) + elif cc_meta.get("kind") == "dsl": from blosc2.dsl_kernel import kernel_from_source dsl_source = cc_meta["dsl_source"] @@ -12520,6 +12546,7 @@ def _load_computed_cols_from_schema(self, schema_dict: dict) -> None: def _load_materialized_cols_from_schema(self, schema_dict: dict) -> None: """Reconstruct ``_materialized_cols`` from persisted metadata.""" + self._check_legacy_dsl_policy(schema_dict) for meta in schema_dict.get("materialized_columns", []): loaded = { "computed_column": meta.get("computed_column"), @@ -12531,12 +12558,53 @@ def _load_materialized_cols_from_schema(self, schema_dict: dict) -> None: } if "transformer" in meta: loaded["transformer"] = dict(meta["transformer"]) + if meta.get("transformer_kind") == "portable": + desc = self._validate_portable_table_metadata(meta) + if self._cols[meta["name"]].dtype != desc["dtype"]: + raise ValueError("Stored portable output dtype disagrees with native signature") + loaded.update( + {key: desc[key] for key in ("artifact", "bindings", "row_domain", "row_shape", "kernel")} + ) if meta.get("dsl_source") is not None: loaded["dsl_source"] = meta["dsl_source"] if meta.get("jit_backend") is not None: loaded["jit_backend"] = meta["jit_backend"] self._materialized_cols[meta["name"]] = loaded + def _check_legacy_dsl_policy(self, schema_dict): + from blosc2.deserialization import DeserializeMode, normalize_deserialize + from blosc2.exceptions import UnsafeDeserializationError + + legacy = any( + cc.get("kind") == "dsl" or "dsl_source" in cc for cc in schema_dict.get("computed_columns", []) + ) or any( + meta.get("dsl_source") is not None or meta.get("transformer_kind") == "dsl" + for meta in schema_dict.get("materialized_columns", []) + ) + storage = self._storage + policy = getattr( + storage, + "_deserialize_mode", + getattr(getattr(storage, "_store", None), "_deserialize_mode", "safe"), + ) + if legacy and normalize_deserialize(policy) is not DeserializeMode.FULL: + raise UnsafeDeserializationError("legacy DSL table columns") + + def _preflight_portable_persistence(self): + for meta in [*self._computed_cols.values(), *self._materialized_cols.values()]: + if meta.get("kind") == "portable" or meta.get("transformer_kind") == "portable": + self._validate_portable_table_metadata(meta) + if any(cc.get("kind") == "dsl" for cc in self._computed_cols.values()) or any( + meta.get("dsl_source") is not None or meta.get("transformer_kind") == "dsl" + for meta in self._materialized_cols.values() + ): + raise TypeError( + "Source-authored vector table columns have no explicit row-domain contract; " + "use add_portable_computed_column or add_portable_generated_column with " + "the DSLKernel, dtype=..., and row_domain='independent'. " + "Legacy source metadata is not automatically migrated." + ) + def _require_computed_column(self, name: str) -> dict: """Return metadata for computed column *name* or raise ``KeyError``.""" try: @@ -12560,6 +12628,9 @@ def _autofill_materialized_row_values(self, row: dict[str, Any]) -> dict[str, An if meta.get("transformer_kind") == "row_transformer": transformer = RowTransformer.from_metadata(meta["transformer"]) row[name] = np.asarray(transformer.evaluate_row(row), dtype=meta["dtype"]) + elif meta.get("transformer_kind") == "portable": + single = {dep: [row[dep]] for dep in meta["col_deps"]} + row[name] = self._evaluate_portable_rows(meta, single, count=1)[0] elif meta.get("transformer_kind") == "dsl": single = {dep: [row[dep]] for dep in meta["col_deps"]} row[name] = self._evaluate_dsl_materialized_batch(meta, single)[0] @@ -12607,6 +12678,8 @@ def _autofill_materialized_batch_columns( if meta.get("transformer_kind") == "row_transformer": transformer = RowTransformer.from_metadata(meta["transformer"]) values = transformer.evaluate_batch(raw_columns) + elif meta.get("transformer_kind") == "portable": + values = self._evaluate_portable_rows(meta, raw_columns) elif meta.get("transformer_kind") == "dsl": values = self._evaluate_dsl_materialized_batch(meta, raw_columns) else: @@ -12785,9 +12858,18 @@ def materialize_computed_column( if target_name in self._computed_cols: raise ValueError(f"A computed column named {target_name!r} already exists.") target_dtype = np.dtype(dtype) if dtype is not None else np.dtype(cc["dtype"]) + if cc.get("kind") == "portable" and target_dtype != cc["dtype"]: + raise ValueError("Portable materialization cannot override its native result dtype") self._create_empty_stored_column(target_name, target_dtype, cparams=cparams) - if cc.get("kind") == "dsl": + if cc.get("kind") == "portable": + self._materialized_cols[target_name] = { + **cc, + "computed_column": name, + "transformer_kind": "portable", + "stale": False, + } + elif cc.get("kind") == "dsl": self._materialized_cols[target_name] = { "computed_column": name, "expression": None, @@ -12941,12 +13023,16 @@ def _normalize_transformer(self, expr, inputs=None) -> dict: identity and *inputs* is ignored). """ if isinstance(expr, blosc2.DSLKernel): + if not isinstance(self._storage, InMemoryTableStorage): + raise TypeError("Persisting full DSL table columns is unsupported; use portable artifacts") kernel, col_deps = self._resolve_dsl_kernel(expr, inputs) self._guard_utf8_kernel_deps(col_deps) return {"kind": "dsl", "kernel": kernel, "col_deps": col_deps} # Resolve a callable once (a lambda may return a LazyExpr or a LazyUDF). obj = expr(self._cols) if (callable(expr) and not isinstance(expr, blosc2.LazyExpr)) else expr if isinstance(obj, blosc2.LazyUDF): + if not isinstance(self._storage, InMemoryTableStorage): + raise TypeError("Persisting full DSL table columns is unsupported; use portable artifacts") if not isinstance(obj.func, blosc2.DSLKernel): raise TypeError( "Only LazyUDFs backed by a @blosc2.dsl_kernel are supported as CTable columns." @@ -13016,6 +13102,13 @@ def _build_computed_lazy(self, cc: dict): (the all-NDArray miniexpr fast path). The full column is recomputed on each access — acceptable for a virtual, unstored column. """ + if cc.get("kind") == "portable": + positions = np.flatnonzero(self._valid_rows[:]) + raw = {dep: self._cols[dep][positions] for dep in cc["col_deps"]} + values = self._evaluate_portable_rows(cc, raw, count=len(positions)) + output = np.zeros(len(self._valid_rows), dtype=cc["dtype"]) + output[positions] = values + return blosc2.asarray(output) if cc.get("kind") == "dsl": operands = tuple(self._cols[d] for d in cc["col_deps"]) return blosc2.lazyudf( @@ -13082,6 +13175,148 @@ def _generated_dependency_closure(self, source: str) -> set[str]: queue.append(name) return affected + def _portable_table_descriptor( + self, kernel, bindings, row_domain, *, dtype=None, cardinality="elementwise" + ): + if isinstance(kernel, blosc2.DSLKernel): + if row_domain != "independent": + raise ValueError("Authored table kernels require explicit row_domain='independent'") + if dtype is None: + raise ValueError("Authored portable table kernels require an explicit output dtype") + bindings = dict(bindings) + if any(dep not in self._cols for dep in bindings.values()): + raise ValueError("Portable row inputs require stored columns") + row_shape = ( + np.broadcast_shapes(*(tuple(self._cols[dep].shape[1:]) or (1,) for dep in bindings.values())) + if bindings + else (1,) + ) + kernel = blosc2.PortableKernel.from_json( + kernel.export( + {name: self._cols[dep].dtype for name, dep in bindings.items()}, + dtype, + version="1.0", + ndim=len(row_shape), + cardinality=cardinality, + ), + jit=False, + ) + if not isinstance(kernel, blosc2.PortableKernel): + raise TypeError("Portable table transformers require a validated PortableKernel") + if row_domain != "independent": + raise ValueError("Only explicit independent-row portable table groups are implemented") + if kernel.schema_version != "1.0": + raise ValueError("Independent rows require portable 1.0") + bindings = dict(bindings) + if set(bindings) != set(kernel.input_dtypes): + raise ValueError("Portable table bindings must match the native signature") + row_shapes = set() + for name, dep in bindings.items(): + if dep not in self._cols: + raise ValueError("Portable row inputs require stored columns") + row_shapes.add(tuple(self._cols[dep].shape[1:]) or (1,)) + if np.dtype(self._cols[dep].dtype) != kernel.input_dtypes[name]: + raise ValueError("Portable table column dtype disagrees with the native signature") + row_shape = np.broadcast_shapes(*row_shapes) if row_shapes else (1,) + if kernel.context_ndim not in (0, len(row_shape)): + raise ValueError("Portable context rank disagrees with the fixed row domain") + if row_shape != (1,) and kernel.result_cardinality != "block_scalar": + raise ValueError("Fixed-shape row transformers currently require scalar native returns") + return { + "kind": "portable", + "kernel": kernel, + "artifact": kernel.to_json(), + "bindings": bindings, + "row_domain": row_domain, + "row_shape": row_shape, + "col_deps": list(dict.fromkeys(bindings.values())), + "dtype": kernel.output_dtype, + } + + def _validate_portable_table_metadata(self, meta): + kernel = blosc2.PortableKernel.from_json(meta["artifact"]) + desc = self._portable_table_descriptor(kernel, meta["bindings"], meta["row_domain"]) + if ( + desc["dtype"] != np.dtype(meta["dtype"]) + or desc["col_deps"] != meta["col_deps"] + or desc["row_shape"] != tuple(meta.get("row_shape", (1,))) + ): + raise ValueError("Portable table metadata disagrees with native signature/bindings") + return desc + + def _evaluate_portable_rows(self, meta, raw_columns, *, count=None): + kernel = meta.get("kernel") + if kernel is None: + kernel = blosc2.PortableKernel.from_json(meta["artifact"]) + arrays = { + name: np.asarray(raw_columns[dep], dtype=kernel.input_dtypes[name]) + for name, dep in meta["bindings"].items() + } + if count is None: + count = len(next(iter(raw_columns.values()))) + if any(len(array) != count for array in arrays.values()): + raise ValueError("Portable row operands disagree on batch length") + result = np.empty(count, dtype=kernel.output_dtype) + row_shape = tuple(meta["row_shape"]) + context = ( + {"logical_shape": row_shape, "block_origin": (0,) * len(row_shape)} + if kernel.context_ndim + else {} + ) + for row in range(count): + # Slice, rather than extract a scalar: NumPy scalar strings discard + # the declared slot width and would invalidate the native signature. + inputs = { + name: np.broadcast_to(array[row : row + 1].reshape(array.shape[1:] or (1,)), row_shape) + for name, array in arrays.items() + } + value = kernel.evaluate_block(inputs, block_shape=row_shape, **context) + result[row] = value[()] if value.shape == () else value[0] + return result + + def add_portable_computed_column( + self, name, kernel, *, inputs, row_domain, dtype=None, cardinality="elementwise" + ): + """Bind a native row transformer with explicit independent grouping. + + Each row's ND domain is (1,), origin (0,); append, refresh, deletion and + compaction do not change group membership or synthesize global row indices. + Fixed-shape row operands use their complete row domain for scalar native + reductions. Multi-row partitions/dynamic global coordinates reject explicitly. + A newly authored DSLKernel is normalized/validated before mutation; supply + dtype and, for row reductions, cardinality='block_scalar'. A PortableKernel + already carries that signature. The older vector-column DSL API does not + declare independent row grouping and is not converted implicitly. + """ + if self.base is not None or self._read_only: + raise ValueError("Portable columns require a writable root table") + _validate_column_name(name) + if name in self._cols or name in self._computed_cols: + raise ValueError("Column already exists") + desc = self._portable_table_descriptor( + kernel, inputs, row_domain, dtype=dtype, cardinality=cardinality + ) + self._computed_cols[name] = desc + self.col_names.append(name) + self._col_widths[name] = max(len(name), 15) + if isinstance(self._storage, FileTableStorage): + self._storage.save_schema(self._schema_dict_with_computed()) + + def add_portable_generated_column( + self, name, kernel, *, inputs, row_domain, dtype=None, cardinality="elementwise" + ): + """Store an independent-row native transformer, retaining it for append/refresh.""" + desc = self._portable_table_descriptor( + kernel, inputs, row_domain, dtype=dtype, cardinality=cardinality + ) + raw = {dep: self[dep][:] for dep in desc["col_deps"]} + values = self._evaluate_portable_rows(desc, raw, count=len(self)) + spec = self._coerce_generated_spec(None, values) + self.add_column(name, spec, values=values) + self._materialized_cols[name] = {**desc, "transformer_kind": "portable", "stale": False} + if isinstance(self._storage, FileTableStorage): + self._storage.save_schema(self._schema_dict_with_computed()) + def _mark_generated_columns_stale(self, source: str) -> None: affected = self._generated_dependency_closure(source) changed = False @@ -13104,6 +13339,9 @@ def refresh_generated_column(self, name: str) -> None: if meta.get("transformer_kind") == "row_transformer": transformer = RowTransformer.from_metadata(meta["transformer"]) values = np.asarray(transformer.evaluate_existing(self), dtype=meta["dtype"]) + elif meta.get("transformer_kind") == "portable": + raw_columns = {dep: self[dep][:] for dep in meta["col_deps"]} + values = self._evaluate_portable_rows(meta, raw_columns, count=len(self)) elif meta.get("transformer_kind") == "dsl": raw_columns = {dep: self[dep][:] for dep in meta["col_deps"]} values = self._evaluate_dsl_materialized_batch(meta, raw_columns) @@ -14893,6 +15131,7 @@ def copy( # noqa: C901 the source. """ if urlpath is not None: + self._preflight_portable_persistence() urlpath = os.fspath(urlpath) if chunks is not None or blocks is not None or cparams is not None: # When storage layout changes we must go through _save_to_storage @@ -15136,18 +15375,10 @@ def _empty_copy( # Rebuild computed columns with the new NDArray objects as operands obj._computed_cols = {} for cc_name, cc in self._computed_cols.items(): - if cc.get("kind") == "dsl": - # DSL entries hold the live kernel; the LazyUDF is rebuilt on - # demand from obj._cols, so no operand rebinding is needed here. - dsl_entry: dict[str, Any] = { - "kind": "dsl", - "dsl_source": cc["dsl_source"], - "kernel": cc["kernel"], - "col_deps": cc["col_deps"], - "dtype": cc["dtype"], - **({"jit_backend": cc["jit_backend"]} if "jit_backend" in cc else {}), - } - obj._computed_cols[cc_name] = dsl_entry + if cc.get("kind") in {"portable", "dsl"}: + # Native/source descriptors resolve columns on demand. Neither + # retains the old table's array operands in a cached expression. + obj._computed_cols[cc_name] = dict(cc) else: operands = {f"o{i}": new_cols[dep] for i, dep in enumerate(cc["col_deps"])} new_lazy = blosc2.lazyexpr(cc["expression"], operands) diff --git a/src/blosc2/deserialization.py b/src/blosc2/deserialization.py index 627163c4e..c71cc4f89 100644 --- a/src/blosc2/deserialization.py +++ b/src/blosc2/deserialization.py @@ -37,6 +37,8 @@ def get_deserialize(obj: Any, default: str | DeserializeMode = DeserializeMode.F """Return the policy attached to *obj* or its physical SChunk carrier.""" carrier = getattr(obj, "schunk", obj) + if carrier is None: + carrier = obj return normalize_deserialize(getattr(carrier, "_deserialize_mode", default)) @@ -45,6 +47,8 @@ def set_deserialize(obj: Any, value: str | DeserializeMode) -> Any: mode = normalize_deserialize(value) carrier = getattr(obj, "schunk", obj) + if carrier is None: + carrier = obj current = getattr(carrier, "_deserialize_mode", None) if current is not None and normalize_deserialize(current) is not mode: raise ValueError( diff --git a/src/blosc2/dict_store.py b/src/blosc2/dict_store.py index b173b6b3b..239d6c388 100644 --- a/src/blosc2/dict_store.py +++ b/src/blosc2/dict_store.py @@ -625,6 +625,9 @@ def __setitem__( self, key: str, value: blosc2.Array | SChunk | blosc2.ObjectArray | blosc2.BatchArray ) -> None: """Add a node to the DictStore.""" + from blosc2.b2objects import preflight_persistence + + preflight_persistence(value) self._modified = True if isinstance(value, np.ndarray): value = blosc2.asarray(value, cparams=self.cparams, dparams=self.dparams) diff --git a/src/blosc2/dsl_artifact_bridge.h b/src/blosc2/dsl_artifact_bridge.h index cfc28fc2a..df1a95476 100644 --- a/src/blosc2/dsl_artifact_bridge.h +++ b/src/blosc2/dsl_artifact_bridge.h @@ -9,10 +9,18 @@ #ifdef B2_HAVE_PORTABLE_ARTIFACT #include "miniexpr_artifact.h" +#endif +#if defined(B2_HAVE_PORTABLE_ARTIFACT) && defined(ME_ARTIFACT_DRAFT_SCHEMA_VERSION) typedef me_artifact_input b2_artifact_input; typedef me_artifact_error b2_artifact_error; -static int b2_artifact_available(void) { return 1; } +static int b2_artifact_available(void) { return !strcmp(ME_ARTIFACT_SCHEMA_VERSION, "1.0"); } static int b2_artifact_load(const char *json, size_t size, int jit, void **out, b2_artifact_error *error) { + if (!b2_artifact_available()) { + *out = NULL; + memset(error, 0, sizeof(*error)); + snprintf(error->message, sizeof(error->message), "Build with miniexpr draft 1.0 artifact support"); + return -100; + } me_artifact *handle = NULL; int rc = me_artifact_load(json, size, (me_jit_mode)jit, &handle, error); *out = handle; @@ -65,4 +73,51 @@ static me_dtype b2_artifact_dtype(const void *handle, int index) { (void)handle; static me_dtype b2_artifact_output(const void *handle) { (void)handle; return ME_AUTO; } static int b2_artifact_jit(const void *handle) { (void)handle; return 0; } #endif +#if defined(B2_HAVE_PORTABLE_ARTIFACT) && defined(ME_ARTIFACT_DRAFT_SCHEMA_VERSION) +typedef me_artifact_buffer b2_artifact_buffer; +typedef me_artifact_eval_descriptor b2_artifact_descriptor; +static int b2_artifact_descriptor_available(void) { return b2_artifact_available(); } +static int b2_artifact_cardinality(const void *handle) { return me_artifact_result_cardinality(handle); } +static size_t b2_artifact_width(const void *handle, int index) { + return index < 0 ? me_artifact_output_itemsize(handle) : me_artifact_input_itemsize(handle, index); +} +static int b2_artifact_rank(const void *handle) { return me_artifact_context_ndim(handle); } +static const char *b2_artifact_version(const void *handle) { return me_artifact_schema_version(handle); } +static int b2_artifact_eval_ex(const void *handle, const b2_artifact_buffer *inputs, int ninputs, + void *output, const b2_artifact_descriptor *descriptor, b2_artifact_error *error) { + return me_artifact_eval_ex(handle, inputs, ninputs, output, descriptor, error); +} +#else +typedef struct { + const char *name; + me_dtype dtype; + size_t itemsize; + const void *data; + size_t capacity; +} b2_artifact_buffer; +typedef struct { + size_t struct_size; + unsigned int version; + size_t nitems; + size_t output_capacity; + const uint8_t *valid_mask; + size_t valid_mask_capacity; + int ndim; + const int64_t *logical_shape; + const int64_t *block_origin; + const int64_t *block_extent; +} b2_artifact_descriptor; +static int b2_artifact_descriptor_available(void) { return 0; } +static int b2_artifact_cardinality(const void *handle) { (void)handle; return 0; } +static size_t b2_artifact_width(const void *handle, int index) { (void)handle; (void)index; return 0; } +static int b2_artifact_rank(const void *handle) { (void)handle; return 0; } +static const char *b2_artifact_version(const void *handle) { (void)handle; return "unsupported"; } +static int b2_artifact_eval_ex(const void *handle, const b2_artifact_buffer *inputs, int ninputs, + void *output, const b2_artifact_descriptor *descriptor, b2_artifact_error *error) { + (void)handle; (void)inputs; (void)ninputs; (void)output; (void)descriptor; + memset(error, 0, sizeof(*error)); + snprintf(error->message, sizeof(error->message), "Build with draft miniexpr 1.0 descriptor support"); + return -2; +} +#endif #endif diff --git a/src/blosc2/dsl_kernel.py b/src/blosc2/dsl_kernel.py index 153e70f45..6a6226e05 100644 --- a/src/blosc2/dsl_kernel.py +++ b/src/blosc2/dsl_kernel.py @@ -934,7 +934,18 @@ def _input_names_from_signature(func_node: ast.FunctionDef) -> list[str]: raise ValueError("DSL kernel does not support *args/**kwargs/kwonly args") return [a.arg for a in (args.posonlyargs + args.args)] - def export(self, input_dtypes, output_dtype, *, capture_dtypes=None, constants=None, metadata=None): + def export( + self, + input_dtypes, + output_dtype, + *, + capture_dtypes=None, + constants=None, + metadata=None, + version="1.0", + cardinality="elementwise", + ndim=0, + ): """Export a typed portable JSON artifact without calling the Python function. ``input_dtypes`` maps runtime parameter names to explicit logical dtypes. @@ -943,8 +954,13 @@ def export(self, input_dtypes, output_dtype, *, capture_dtypes=None, constants=N collision-free parameters. ``capture_dtypes`` overrides scalar type inference (Python bool/int/float infer bool/int64/float64 respectively; supported NumPy scalars retain their dtype). Arrays, objects, and external - callbacks cannot be captures. Profile 0.1 requires homogeneous parameter - dtypes. Import via :meth:`blosc2.PortableKernel.from_json`. + callbacks cannot be captures. Import via :meth:`blosc2.PortableKernel.from_json`. + + The default ``version="1.0"`` is an experimental implementation draft. It + supports mixed numeric types and fixed byte/Unicode string snapshots; + ``cardinality`` is ``elementwise`` or ``block_scalar``, and ``ndim`` + declares the required logical coordinate rank. The native loader + validates normalized source, widths and cardinality before export succeeds. """ from .portable_kernel import export_portable_kernel @@ -955,6 +971,9 @@ def export(self, input_dtypes, output_dtype, *, capture_dtypes=None, constants=N capture_dtypes=capture_dtypes, constants=constants, metadata=metadata, + version=version, + cardinality=cardinality, + ndim=ndim, ) def __call__(self, inputs_tuple, output, offset=None): @@ -1049,8 +1068,10 @@ def validate_dsl(func): } -def validate_portable_dsl(source, input_dtypes, output_dtype, *, language_version="0.1"): - """Check raw native source and a typed signature against conservative portable profile 0.1. +def validate_portable_dsl( + source, input_dtypes, output_dtype, *, language_version="1.0", ndim=0, cardinality=None +): + """Check raw native source and an exact typed signature against a portable profile. Return a dictionary with ``valid``, ``status``, ``line``, ``column``, and ``error``. Validation neither executes a kernel nor invokes a JIT compiler. @@ -1061,6 +1082,13 @@ def validate_portable_dsl(source, input_dtypes, output_dtype, *, language_versio ``input_dtypes`` maps parameter names to explicit dtype descriptions. Parameter binding is by name; mapping order may differ from source order. Dtypes describe logical values, not the storage layout of any actual input arrays. + The default ``language_version="1.0"`` is an experimental implementation + draft admitting mixed numeric and fixed-width ``S``/``U`` + signatures. ``ndim`` declares the logical rank for reserved ND symbols; + missing or insufficient context rejects explicitly. ``cardinality`` may be + ``elementwise`` or ``block_scalar`` to assert the native return contract; + None validates the inferred contract. Neither arrays nor an artifact JSON + loader are needed. Exact string output width is checked natively. """ from collections.abc import Mapping @@ -1072,8 +1100,21 @@ def validate_portable_dsl(source, input_dtypes, output_dtype, *, language_versio raise TypeError("input_dtypes must map parameter names to explicit dtypes") if "\x00" in source or "\x00" in language_version: raise ValueError("Portable DSL source and version must not contain NUL characters") + if language_version != "1.0": + return { + "valid": False, + "status": "unsupported_version", + "line": 0, + "column": 0, + "error": "Unsupported portable DSL version; expected draft 1.0", + } if any(not isinstance(name, str) or "\x00" in name for name in input_dtypes): raise ValueError("Portable DSL input names must be strings without NUL characters") + if type(ndim) is not int or not 0 <= ndim <= 2**31 - 1: + raise ValueError("ndim must be a nonnegative Python integer representable by the native API") + contracts = {None: 0, "elementwise": 1, "block_scalar": 2} + if cardinality not in contracts: + raise ValueError("cardinality must be None, 'elementwise' or 'block_scalar'") validator = getattr(blosc2_ext, "validate_portable_dsl_source", None) if validator is None: return { @@ -1083,7 +1124,7 @@ def validate_portable_dsl(source, input_dtypes, output_dtype, *, language_versio "column": 0, "error": "Rebuild miniexpr and the extension with portable DSL validation support", } - return validator(source, input_dtypes, output_dtype, language_version) + return validator(source, input_dtypes, output_dtype, language_version, ndim, contracts[cardinality]) def validate_dsl_jit(func, operands, out_dtype, *, shape=(64,), chunks=None, blocks=None): diff --git a/src/blosc2/embed_store.py b/src/blosc2/embed_store.py index e91abacc7..582a3c265 100644 --- a/src/blosc2/embed_store.py +++ b/src/blosc2/embed_store.py @@ -273,6 +273,9 @@ def __setitem__( self, key: str, value: blosc2.Array | SChunk | blosc2.ObjectArray | blosc2.BatchArray ) -> None: """Add a node to the embed store.""" + from blosc2.b2objects import preflight_persistence + + preflight_persistence(value) if self.mode == "r": raise ValueError("Cannot set items in read-only mode.") with self._write_bracket(): diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py index 9f94f5761..44945f63f 100644 --- a/src/blosc2/lazyexpr.py +++ b/src/blosc2/lazyexpr.py @@ -14,7 +14,6 @@ import contextlib import copy import enum -import inspect import linecache import math import operator @@ -22,7 +21,6 @@ import pathlib import re import sys -import textwrap import threading from abc import ABC, abstractmethod, abstractproperty from collections.abc import MutableMapping @@ -5106,6 +5104,10 @@ def __init__( def dtype(self): return self._dtype + @property + def nbytes(self): + return math.prod(self.shape) * np.dtype(self.dtype).itemsize + @property def ndim(self) -> int: return len(self.shape) @@ -5296,65 +5298,20 @@ def save(self, urlpath=None, **kwargs): ----- * All operands must be :ref:`NDArray` or :ref:`C2Array` objects stored on disk or a remote server (i.e. they must have a ``urlpath``). - * When the :ref:`LazyUDF` wraps a :func:`blosc2.dsl_kernel`-decorated - function, the DSL source is preserved verbatim in the saved metadata. - On reload via :func:`blosc2.open`, the function is restored as a full - :class:`~blosc2.dsl_kernel.DSLKernel` so the miniexpr JIT fast path - remains available without any extra work from the caller. + * Newly authored DSLKernel functions normalize and validate as portable + 1.0 before saving. Python UDFs and noncompliant kernels reject before + writing. Block-scalar kernels require the explicit portable API. + Historical source-backed files require ``deserialize='full'`` on open + and are not automatically migrated by saving. """ if urlpath is None: raise ValueError("To save a LazyArray you must provide an urlpath") - kwargs["urlpath"] = urlpath - kwargs["mode"] = "w" # always overwrite the file in urlpath - try: - self._to_b2object_carrier(**kwargs) - except (TypeError, ValueError): - meta = kwargs.get("meta", {}) - meta["LazyArray"] = LazyArrayEnum.UDF.value - kwargs["meta"] = meta - - # Create an empty array; useful for providing the shape and dtype of the outcome - array = blosc2.empty(shape=self.shape, dtype=self.dtype, **kwargs) + from blosc2.portable_lazy import portable_from_lazyudf - # Save the expression and operands in the metadata - operands = {} - operands_ = self.inputs_dict - for i, (_key, value) in enumerate(operands_.items()): - pos_key = f"o{i}" # always use positional keys for consistent loading - if isinstance(value, blosc2.C2Array): - operands[pos_key] = { - "path": str(value.path), - "urlbase": value.urlbase, - } - continue - if isinstance(value, blosc2.Proxy): - # Take the required info from the Proxy._cache container - value = value._cache - if not hasattr(value, "schunk"): - raise ValueError( - "To save a LazyArray, all operands must be blosc2.NDArray or blosc2.C2Array objects" - ) from None - if value.schunk.urlpath is None: - raise ValueError( - "To save a LazyArray, all operands must be stored on disk/network" - ) from None - operands[pos_key] = value.schunk.urlpath - udf_func = self.func.func if isinstance(self.func, DSLKernel) else self.func - udf_name = getattr(udf_func, "__name__", self.func.__name__) - try: - udf_source = textwrap.dedent(inspect.getsource(udf_func)).lstrip() - except Exception: - udf_source = None - meta = { - "UDF": udf_source, - "operands": operands, - "name": udf_name, - } - if isinstance(self.func, DSLKernel) and self.func.dsl_source is not None: - meta["dsl_source"] = self.func.dsl_source - array.schunk.vlmeta["_LazyArray"] = meta - write_b2object_user_vlmeta(array, self._get_user_vlmeta()) + portable = portable_from_lazyudf(self) + portable.save(urlpath, **kwargs) + self.array, self.schunk = portable.array, portable.schunk def to_cframe(self) -> bytes: return self._to_b2object_carrier().to_cframe() @@ -5381,8 +5338,12 @@ def _to_b2object_carrier(self, **kwargs): ref_urlpath = operand_urlpath.as_posix() operand_payload["urlpath"] = ref_urlpath + from blosc2.msgpack_utils import msgpack_packb + + msgpack_packb(payload) + msgpack_packb(self._get_user_vlmeta()) array = make_b2object_carrier( - "lazyudf", + "portable", self.shape, self.dtype, chunks=self.chunks, @@ -5752,6 +5713,12 @@ def _reconstruct_lazyudf(expr, lazyarray, operands_dict, array): def open_lazyarray(array): + from blosc2.deserialization import DeserializeMode, get_deserialize + from blosc2.exceptions import UnsafeDeserializationError + + policy = get_deserialize(array) + if policy is not DeserializeMode.FULL: + raise UnsafeDeserializationError("legacy LazyArray") value = array.schunk.meta["LazyArray"] lazyarray = array.schunk.vlmeta["_LazyArray"] if value == LazyArrayEnum.Expr.value: @@ -5769,7 +5736,7 @@ def open_lazyarray(array): if isinstance(v, str): v = parent_path / v try: - op = blosc2.open(v, mode="r") + op = blosc2.open(v, mode="r", deserialize=policy) except FileNotFoundError: missing_ops[key] = v else: @@ -5794,6 +5761,7 @@ def open_lazyarray(array): new_expr = LazyExpr._new_expr(expr, operands_dict, guess=True, out=None, where=None) elif value == LazyArrayEnum.UDF.value: new_expr = _reconstruct_lazyudf(expr, lazyarray, operands_dict, array) + new_expr._legacy_source_recipe = True # Make the array info available for the user (only available when opened from disk) new_expr.array = array diff --git a/src/blosc2/msgpack_utils.py b/src/blosc2/msgpack_utils.py index b95890fcf..2854bb0ab 100644 --- a/src/blosc2/msgpack_utils.py +++ b/src/blosc2/msgpack_utils.py @@ -48,7 +48,7 @@ def _encode_structured_reference(obj): return None -def _decode_structured_reference(data): +def _decode_structured_reference(data, *, deserialize="safe"): payload = unpackb(data) if not isinstance(payload, dict): raise TypeError("Structured Blosc2 msgpack payload must decode to a mapping") @@ -61,8 +61,8 @@ def _decode_structured_reference(data): if kind == "ref": ref_payload = payload.get("ref") return Ref.from_dict(ref_payload) - if kind in {"c2array", "lazyexpr", "lazyudf"}: - return decode_b2object_payload(payload) + if kind in {"c2array", "lazyexpr", "lazyudf", "portable"}: + return decode_b2object_payload(payload, deserialize=deserialize) raise ValueError(f"Unsupported structured Blosc2 msgpack payload kind: {kind!r}") @@ -80,7 +80,7 @@ def _encode_ndarray(value): return ExtType(_BLOSC2_NDARRAY_EXT_CODE, msgpack_packb(payload)) -def _decode_ndarray(data, *, deserialize=DeserializeMode.FULL): +def _decode_ndarray(data, *, deserialize=DeserializeMode.SAFE): from blosc2.hdf5_source import dtype_from_value payload = msgpack_unpackb(data, deserialize=deserialize) @@ -139,34 +139,38 @@ def decode_tuple_list_hook(obj): return obj -def _decode_msgpack_ext(code, data): +def _decode_msgpack_ext(code, data, *, deserialize=DeserializeMode.SAFE): import blosc2 if code == _BLOSC2_EXT_CODE: - return blosc2.from_cframe(data, copy=True, deserialize="full") + return blosc2.from_cframe(data, copy=True, deserialize=deserialize) if code == _BLOSC2_STRUCTURED_EXT_CODE: - return _decode_structured_reference(data) + return _decode_structured_reference(data, deserialize=deserialize) if code == _BLOSC2_COMPLEX_EXT_CODE: real, imag = struct.unpack(">dd", data) return complex(real, imag) if code == _BLOSC2_NDARRAY_EXT_CODE: - return _decode_ndarray(data, deserialize=DeserializeMode.FULL) + return _decode_ndarray(data, deserialize=deserialize) if code == _BLOSC2_SET_EXT_CODE: - return set(msgpack_unpackb(data)) + return set(msgpack_unpackb(data, deserialize=deserialize)) return ExtType(code, data) -def msgpack_unpackb(payload, *, deserialize=DeserializeMode.FULL): +def msgpack_unpackb(payload, *, deserialize=DeserializeMode.SAFE): """Decode a Blosc2 MessagePack value under the selected policy. - This low-level helper retains ``"full"`` as its default for compatibility. - APIs accepting persisted input pass their safe-by-default policy explicitly. + Active reconstruction requires explicit ``deserialize="full"``. Container + readers pass their effective policy through recursive values unchanged. """ mode = normalize_deserialize(deserialize) if mode is DeserializeMode.SAFE: return _safe_msgpack_unpackb(payload) - return unpackb(payload, list_hook=decode_tuple_list_hook, ext_hook=_decode_msgpack_ext) + return unpackb( + payload, + list_hook=decode_tuple_list_hook, + ext_hook=lambda code, data: _decode_msgpack_ext(code, data, deserialize=mode), + ) def _safe_msgpack_unpackb(payload): @@ -205,12 +209,15 @@ def decode_ext(code, data): if code == _BLOSC2_EXT_CODE: kind = "embedded cframe" elif code == _BLOSC2_STRUCTURED_EXT_CODE: + candidate = None try: envelope = unpackb(data) candidate = envelope.get("kind") if isinstance(envelope, dict) else None kind = candidate if isinstance(candidate, str) else "structured reference" except Exception: kind = "structured reference" + if candidate in {"portable", "lazyexpr"}: + return decode_b2object_payload(envelope, deserialize="safe") else: kind = f"MessagePack extension code {code}" raise UnsafeDeserializationError(str(kind)) diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index 98ab50b0a..514a1a292 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -5003,6 +5003,9 @@ def to_cframe(self) -> bytes: >>> print("Data type of the NDArray:", blosc_array.dtype) Data type of the NDArray: int32 """ + from blosc2.b2objects import preflight_persistence + + preflight_persistence(self) return super().to_cframe() def copy(self, dtype: np.dtype | str = None, **kwargs: Any) -> NDArray: @@ -5097,6 +5100,9 @@ def save(self, urlpath: str, contiguous=True, **kwargs: Any) -> None: >>> # Save the array to a file >>> a.save("array.b2frame") """ + from blosc2.b2objects import preflight_persistence + + preflight_persistence(self) urlpath = normalize_urlpath(urlpath) if is_fsspec_url(urlpath): if not contiguous: diff --git a/src/blosc2/portable_kernel.py b/src/blosc2/portable_kernel.py index bdc739a56..b55651674 100644 --- a/src/blosc2/portable_kernel.py +++ b/src/blosc2/portable_kernel.py @@ -2,7 +2,7 @@ # Copyright (c) 2026, Blosc Development Team # SPDX-License-Identifier: BSD-3-Clause ####################################################################### -"""Conservative portable DSL 0.1 authoring and native-only execution.""" +"""Portable DSL authoring and native-only execution (draft descriptors when installed).""" from __future__ import annotations @@ -17,10 +17,6 @@ import numpy as np -_DTYPES = {"bool", "int32", "int64", "float32", "float64"} -_SCALAR_TYPES = (bool, int, float, np.bool_, np.int32, np.int64, np.float32, np.float64) -_CALLS = {"sin", "cos", "int", "float", "bool", "range"} - class PortableArtifactError(ValueError): """Artifact failure with a stable category and optional native/source diagnostics.""" @@ -34,13 +30,6 @@ def __init__(self, message, *, status, native_status=0, line=0, column=0): super().__init__(f"{status}{location}: {message}") -def _logical_dtype(value): - dtype = np.dtype(value) - if dtype.name not in _DTYPES or dtype.subdtype or dtype.fields: - raise PortableArtifactError(f"Unsupported portable dtype {dtype}", status="unsupported_requirement") - return dtype.newbyteorder("=") - - def _mapping(value, label): if not isinstance(value, Mapping): raise TypeError(f"{label} must be a mapping") @@ -49,55 +38,100 @@ def _mapping(value, label): return dict(value) -def _encode_scalar(name, value, dtype=None): - # Exact types only: no ndarray.item(), user __float__/__int__, or other hooks. - if type(value) not in _SCALAR_TYPES: - raise PortableArtifactError( - f"Capture {name!r} must be a supported Python/NumPy scalar, not {type(value).__name__}", - status="unsupported_requirement", - ) +def portable_dtype_descriptor(value): + dtype = np.dtype(value).newbyteorder("=") + if ( + dtype.fields + or dtype.subdtype + or dtype.kind not in "biufSU" + or (dtype.kind == "f" and dtype.itemsize not in (4, 8)) + ): + raise PortableArtifactError(f"Unsupported portable dtype {dtype}", status="unsupported_requirement") + if dtype.kind in "SU": + if not dtype.itemsize: + raise PortableArtifactError("Fixed strings require a positive width", status="binding_error") + return {"dtype": "bytes" if dtype.kind == "S" else "unicode32", "itemsize": dtype.itemsize} + return {"dtype": dtype.name} + + +def portable_scalar_descriptor(name, value, dtype=None): + numeric = ( + bool, + int, + float, + np.bool_, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + np.float32, + np.float64, + ) + if type(value) not in numeric + (str, bytes, np.str_, np.bytes_): + raise PortableArtifactError("Captures must be plain numeric/string scalars", status="binding_error") if dtype is None: - if type(value) is bool: - dtype = "bool" - elif type(value) is int: - dtype = "int64" - elif type(value) is float: - dtype = "float64" - else: - dtype = value.dtype - dtype = _logical_dtype(dtype) - if dtype.kind in "bi": - if type(value) in (float, np.float32, np.float64): - raise PortableArtifactError( - f"Capture {name!r} cannot implicitly convert a float to {dtype}", status="binding_error" - ) + dtype = {bool: "bool", int: "int64", float: "float64"}.get(type(value), np.asarray(value).dtype) + descriptor = portable_dtype_descriptor(dtype) + dtype = np.dtype(dtype).newbyteorder("=") + if dtype.kind in "SU": + if (dtype.kind == "S" and type(value) not in (bytes, np.bytes_)) or ( + dtype.kind == "U" and type(value) not in (str, np.str_) + ): + raise PortableArtifactError("String capture family mismatch", status="binding_error") + if len(value) > dtype.itemsize // (4 if dtype.kind == "U" else 1): + raise PortableArtifactError("String capture exceeds its fixed width", status="binding_error") + if dtype.kind == "U" and any(0xD800 <= ord(c) <= 0xDFFF for c in value): + raise PortableArtifactError("Invalid Unicode scalar", status="binding_error") + encoded = np.asarray(value, dtype=dtype).astype(dtype.newbyteorder(">"), copy=False).tobytes().hex() + return { + "name": name, + **descriptor, + "encoding": "bytes-hex" if dtype.kind == "S" else "unicode32be-hex", + "value": encoded, + } + if dtype.kind in "biu": + if type(value) in (float, np.float32, np.float64, str, bytes, np.str_, np.bytes_): + raise PortableArtifactError("Integral captures require integral values", status="binding_error") integer = int(value) - if dtype.kind == "b": - if integer not in (0, 1): - raise PortableArtifactError(f"Capture {name!r} is not Boolean", status="binding_error") - encoding, encoded = "boolean", bool(integer) - else: - limits = np.iinfo(dtype) - if not limits.min <= integer <= limits.max: - raise PortableArtifactError( - f"Capture {name!r} is out of range for {dtype}", status="binding_error" - ) - encoding, encoded = "decimal", str(integer) - else: - maximum = float(np.finfo(dtype).max) - if type(value) in (bool, int, np.bool_, np.int32, np.int64): - integer = int(value) - # Reject precision loss, rather than standardizing Python literal rounding. - if abs(integer) > maximum or int(np.array(integer, dtype=dtype)) != integer: - raise PortableArtifactError( - f"Capture {name!r} is not exactly representable as {dtype}", status="binding_error" - ) - elif math.isfinite(value) and abs(float(value)) > maximum: - raise PortableArtifactError(f"Capture {name!r} overflows {dtype}", status="binding_error") - scalar = np.array(value, dtype=dtype) - encoding = "ieee754-hex" - encoded = scalar.astype(dtype.newbyteorder(">"), copy=False).tobytes().hex() - return {"name": name, "dtype": dtype.name, "encoding": encoding, "value": encoded} + lower, upper = (0, 1) if dtype.kind == "b" else (np.iinfo(dtype).min, np.iinfo(dtype).max) + if not lower <= integer <= upper: + raise PortableArtifactError("Capture is outside its dtype", status="binding_error") + return { + "name": name, + **descriptor, + "encoding": "boolean" if dtype.kind == "b" else "decimal", + "value": bool(integer) if dtype.kind == "b" else str(integer), + } + if type(value) in (str, bytes, np.str_, np.bytes_): + raise PortableArtifactError("Numeric capture family mismatch", status="binding_error") + integral = type(value) in ( + bool, + int, + np.bool_, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + ) + # NumPy signed minima cannot be negated in their own fixed-width dtype. + magnitude = abs(int(value)) if integral else abs(float(value)) + if (integral or math.isfinite(value)) and magnitude > float(np.finfo(dtype).max): + raise PortableArtifactError("Floating capture overflows its dtype", status="binding_error") + scalar = np.asarray(value, dtype=dtype) + return { + "name": name, + **descriptor, + "encoding": "ieee754-hex", + "value": scalar.astype(dtype.newbyteorder(">"), copy=False).tobytes().hex(), + } def _parameterize_source(source, replacements, parameters): @@ -145,17 +179,6 @@ def _capture_scope(func): return scope -def _check_capture_calls(calls, local, scope): - for call in sorted(calls): - if call not in _CALLS or call in local: - raise PortableArtifactError(f"Unsupported call {call!r}", status="unsupported_requirement") - allowed = (getattr(builtins, call, None), getattr(np, call, None), getattr(math, call, None)) - if call in scope and not any(value is not None and scope[call] is value for value in allowed): - raise PortableArtifactError( - f"External callback {call!r} cannot be exported", status="unsupported_requirement" - ) - - def _export_captures(func, source, names, capture_dtypes): scope = _capture_scope(func) tree = ast.parse(source) @@ -167,12 +190,22 @@ def _export_captures(func, source, names, capture_dtypes): calls = { node.func.id for node in nodes if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) } - _check_capture_calls(calls, local, scope) + # Native compilation owns builtin membership. Never export external callbacks. + if any(isinstance(n, (ast.Attribute, ast.Subscript, ast.Lambda)) for n in nodes): + raise PortableArtifactError("Unsupported dynamic authoring syntax", status="unsupported_requirement") + for call in calls: + allowed = (getattr(builtins, call, None), getattr(np, call, None), getattr(math, call, None)) + if call in local or (call in scope and not any(v is not None and scope[call] is v for v in allowed)): + raise PortableArtifactError( + f"External callback {call!r} cannot be exported", status="unsupported_requirement" + ) loads = {node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)} occupied = local | loads | {function.name} replacements = {} encoded = [] for name in sorted(loads - local - calls): + if name in {"_flat_idx", "_ndim"} or (name.startswith(("_i", "_n")) and name[2:].isdigit()): + continue if name not in scope: node = next(node for node in nodes if isinstance(node, ast.Name) and node.id == name) raise PortableArtifactError( @@ -187,7 +220,7 @@ def _export_captures(func, source, names, capture_dtypes): index += 1 candidate = f"_capture_{index}" occupied.add(candidate) - scalar = _encode_scalar(name, scope[name], capture_dtypes.get(name)) + scalar = portable_scalar_descriptor(name, scope[name], capture_dtypes.get(name)) scalar["name"] = candidate encoded.append(scalar) replacements[name] = candidate @@ -202,64 +235,66 @@ def _export_captures(func, source, names, capture_dtypes): def export_portable_kernel( - kernel, input_dtypes, output_dtype, *, capture_dtypes=None, constants=None, metadata=None + kernel, + input_dtypes, + output_dtype, + *, + capture_dtypes=None, + constants=None, + metadata=None, + version="1.0", + cardinality="elementwise", + ndim=0, ): """Implementation of :meth:`blosc2.DSLKernel.export`; does not call the kernel.""" - from .dsl_kernel import DSLKernel, validate_portable_dsl - + if version != "1.0": + raise PortableArtifactError("Unsupported artifact version", status="unsupported_version") inputs = _mapping(input_dtypes, "input_dtypes") constants = _mapping({} if constants is None else constants, "constants") - capture_dtypes = _mapping({} if capture_dtypes is None else capture_dtypes, "capture_dtypes") + captures = _mapping({} if capture_dtypes is None else capture_dtypes, "capture_dtypes") source = kernel.dsl_source if not isinstance(source, str): - raise PortableArtifactError("Kernel has no exportable DSL source", status="invalid_source") - # This is only a header check; native validation remains authoritative for the body. - header = DSLKernel.from_source(source) - names = header.input_names + raise PortableArtifactError("Kernel has no normalized source", status="invalid_source") + names = kernel.input_names if set(inputs) & set(constants) or set(inputs) | set(constants) != set(names): raise PortableArtifactError( "Every parameter needs exactly one input or constant binding", status="binding_error" ) - input_types = {name: _logical_dtype(inputs[name]).name for name in names if name in inputs} - result_type = _logical_dtype(output_dtype).name - encoded = [_encode_scalar(name, value, capture_dtypes.get(name)) for name, value in constants.items()] - used_capture_types = set(constants) & set(capture_dtypes) - + encoded = [ + portable_scalar_descriptor(name, value, captures.get(name)) for name, value in constants.items() + ] + used = set(constants) & set(captures) if kernel.func is not None: - source, captures, used_types = _export_captures(kernel.func, source, names, capture_dtypes) - encoded.extend(captures) - used_capture_types.update(used_types) - unused = set(capture_dtypes) - used_capture_types - if unused: - raise PortableArtifactError(f"Unused capture dtype names: {sorted(unused)}", status="binding_error") - signature = input_types | {item["name"]: item["dtype"] for item in encoded} - info = validate_portable_dsl(source, signature, result_type) - if not info["valid"]: - if info["status"] == "runtime_unsupported": - raise NotImplementedError(info["error"]) - raise PortableArtifactError( - info["error"], status=info["status"], line=info["line"], column=info["column"] - ) + source, captured, extra = _export_captures(kernel.func, source, names, captures) + encoded.extend(captured) + used |= extra + if set(captures) - used: + raise PortableArtifactError("Unused capture dtype names", status="binding_error") + if cardinality not in {"elementwise", "block_scalar"} or type(ndim) is not int or ndim < 0: + raise PortableArtifactError("Invalid return/context contract", status="binding_error") if metadata is not None and not isinstance(metadata, Mapping): raise TypeError("metadata must be a mapping") + # This finite superset grants no unsupported operations: native typed import + # validates the entire normalized source before this export can succeed. manifest = { - "schema_version": "0.1", - "language": {"name": "miniexpr", "version": "0.1"}, - "requires": ["core-scalar"], + "schema_version": "1.0", + "language": {"name": "miniexpr", "version": "1.0"}, + "requires": ["numeric", "control-flow", "block-reductions", "fixed-strings", "nd-context"], "source": source, - "entry_point": header.__name__, - "inputs": [{"name": name, "dtype": dtype} for name, dtype in input_types.items()], + "entry_point": kernel.__name__, + "inputs": [ + {"name": name, **portable_dtype_descriptor(inputs[name])} for name in names if name in inputs + ], "constants": sorted(encoded, key=lambda item: item["name"]), - "output": {"dtype": result_type, "contract": "scalar-per-element"}, + "output": {**portable_dtype_descriptor(output_dtype), "contract": cardinality}, "semantics": {"fp": "strict"}, + "context": {"ndim": ndim}, + "metadata": dict(metadata or {}), } - if metadata is not None: - manifest["metadata"] = dict(metadata) artifact = ( json.dumps(manifest, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + "\n" ) - # Use the same loader as import/C, including bounds and metadata validation. PortableKernel.from_json(artifact, jit=False) return artifact @@ -268,8 +303,9 @@ class PortableKernel: """An owned, typed native artifact; no originating function or Python fallback. Use ``from_json`` to import and ``evaluate`` to return NumPy values. Inputs - are same-shaped logical arrays, bound by name. The conservative 0.1 profile - is enforced by the native adapter (enabled by default in package builds). + are same-shaped logical arrays, bound by name. The installed native adapter + enforces its supported profiles. Draft 1.0 descriptor blocks are available + only when that runtime exposes the extended ABI; no sibling build is inferred. """ @classmethod @@ -294,6 +330,8 @@ def from_json(cls, artifact, *, jit=None): instance._handle = handle instance._artifact = data instance._info = handle.info() + if getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(): + instance._info.update(handle.descriptor_info()) return instance @property @@ -316,6 +354,18 @@ def output_dtype(self): def has_jit(self): return self._info["jit"] + @property + def schema_version(self): + return self._info.get("schema_version", "1.0") + + @property + def result_cardinality(self): + return self._info.get("cardinality", "elementwise") + + @property + def context_ndim(self): + return self._info.get("ndim", 0) + def to_json(self): """Return the original validated JSON text, without dropping metadata.""" return self._artifact.decode("utf-8") @@ -326,6 +376,8 @@ def evaluate(self, inputs, *, shape=None): No implicit dtype conversion or array broadcasting is allowed. Host endian, alignment, and strides are normalized into temporary contiguous buffers. Constant-only kernels require an explicit output ``shape`` (possibly empty). + Draft block-scalar kernels return a scalar NumPy array. ND kernels require + ``evaluate_block`` with explicit logical context. """ inputs = _mapping(inputs, "inputs") if set(inputs) != set(self.input_dtypes): @@ -354,4 +406,60 @@ def evaluate(self, inputs, *, shape=None): arrays[name] = np.require(array, dtype=dtype, requirements=["C", "A"]) if shape is None: raise ValueError("Constant-only kernels require an explicit shape") - return self._handle.evaluate(arrays, shape) + if self.context_ndim: + raise ValueError( + "Use evaluate_block with explicit logical_shape and block_origin for ND artifacts" + ) + return self._handle.evaluate_block(arrays, shape) + + def evaluate_block( + self, inputs, *, block_shape=None, logical_shape=None, block_origin=None, valid_mask=None + ): + """Evaluate one explicit draft 1.0 block, returning its declared cardinality. + + This is standalone block execution, not lazy block-grid scheduling. Native + validation owns coordinates, participating masks and result allocation. + """ + if self.schema_version != "1.0": + raise NotImplementedError("Explicit descriptor blocks require an installed draft 1.0 runtime") + inputs = _mapping(inputs, "inputs") + if set(inputs) != set(self.input_dtypes): + raise PortableArtifactError("Missing or extra runtime inputs", status="binding_error") + if isinstance(block_shape, int): + block_shape = (block_shape,) + elif block_shape is not None: + block_shape = tuple(block_shape) + if block_shape is not None and any(type(size) is not int or size < 0 for size in block_shape): + raise ValueError("block_shape must contain nonnegative Python integers") + arrays = {} + for name, value in inputs.items(): + array = np.asarray(value) + dtype = self.input_dtypes[name] + if array.dtype.newbyteorder("=") != dtype: + raise PortableArtifactError( + f"Input {name!r} must have dtype {dtype}, not {array.dtype}", status="binding_error" + ) + if block_shape is None: + block_shape = array.shape + if array.shape != block_shape: + raise PortableArtifactError( + "Input shape disagrees with block extent", status="binding_error" + ) + arrays[name] = np.require(array, dtype=dtype, requirements=["C", "A"]) + if block_shape is None: + raise ValueError("Constant-only kernels require an explicit block_shape") + return self._handle.evaluate_block( + arrays, + block_shape, + logical_shape=logical_shape, + block_origin=block_origin, + valid_mask=valid_mask, + ) + + def lazy(self, inputs, *, shape=None, partitions=None): + """Bind native inputs to an immutable logical block grid, not a storage grid.""" + from .portable_lazy import PortableLazyArray + + if self.schema_version != "1.0": + raise NotImplementedError("Logical portable partitions require the descriptor profile") + return PortableLazyArray(self, inputs, shape=shape, partitions=partitions) diff --git a/src/blosc2/portable_lazy.py b/src/blosc2/portable_lazy.py new file mode 100644 index 000000000..dc7dcaef4 --- /dev/null +++ b/src/blosc2/portable_lazy.py @@ -0,0 +1,257 @@ +"""Native portable kernels over immutable logical partitions, independent of storage tiles.""" + +from itertools import product +from math import prod + +import numpy as np + +import blosc2 +from blosc2.lazyexpr import LazyArray + + +def portable_from_lazyudf(obj): + """Preflight a newly authored DSL recipe without executing it or touching storage.""" + from blosc2.dsl_kernel import DSLKernel + from blosc2.portable_kernel import PortableArtifactError, PortableKernel + + if not isinstance(obj.func, DSLKernel): + raise TypeError("Python LazyUDF cannot be persisted; use a compliant DSLKernel or PortableKernel") + schunk = getattr(obj, "schunk", None) + if getattr(obj, "_legacy_source_recipe", False) or ( + schunk is not None + and (schunk.meta.get("LazyArray") is not None or schunk.meta.get("b2o", {}).get("kind") == "lazyudf") + ): + raise TypeError("Persisting legacy source-backed LazyUDF is unsupported; author a new DSLKernel") + inputs, constants = {}, {} + for name, value in obj.inputs_dict.items(): + if np.isscalar(value): + constants[name] = value + else: + inputs[name] = value + shape = tuple(int(n) for n in obj.shape) + partitions = tuple(int(n) for n in obj.blocks) + chunks = tuple(int(n) for n in obj.chunks) + if any(c % b for c, b in zip(chunks, partitions, strict=True)): + raise ValueError( + "LazyUDF chunk boundaries split its logical block grid; use PortableKernel.lazy " + "with explicit partitions" + ) + try: + artifact = obj.func.export( + {name: value.dtype for name, value in inputs.items()}, + obj.dtype, + version="1.0", + constants=constants, + ndim=len(shape), + cardinality="elementwise", + ) + except PortableArtifactError as error: + if "return cardinality" in str(error): + raise ValueError( + "LazyUDF declares elementwise shape but the kernel returns a block scalar; " + "use DSLKernel.export(cardinality='block_scalar', version='1.0') and " + "PortableKernel.lazy with explicit partitions" + ) from error + raise + portable = PortableLazyArray( + PortableKernel.from_json(artifact, jit=False), inputs, shape=shape, partitions=partitions + ) + portable._set_user_vlmeta(obj._get_user_vlmeta(), sync=False) + return portable + + +class PortableLazyArray(LazyArray): + def __init__(self, kernel, inputs, *, shape=None, partitions=None): + self.kernel = kernel + self.inputs = dict(inputs) + if set(self.inputs) != set(kernel.input_dtypes): + raise ValueError("Portable input names disagree with native signature") + if shape is None: + if not self.inputs: + raise ValueError("Constant-only portable arrays require shape") + shape = np.broadcast_shapes(*(value.shape for value in self.inputs.values())) + self._domain = tuple(shape) + if any(type(n) is not int or not 0 <= n <= np.iinfo(np.int64).max for n in self._domain): + raise ValueError("Logical shape must contain nonnegative integers") + if kernel.context_ndim and kernel.context_ndim != len(self._domain): + raise ValueError("Artifact context rank disagrees with logical domain") + for name, value in self.inputs.items(): + if ( + np.broadcast_shapes(tuple(value.shape), self._domain) != self._domain + or np.dtype(value.dtype).newbyteorder("=") != kernel.input_dtypes[name] + ): + raise ValueError( + "Portable operands must broadcast to the logical domain and match native dtype" + ) + self._partitions = tuple(partitions if partitions is not None else (max(n, 1) for n in self._domain)) + if len(self._partitions) != len(self._domain) or any( + type(n) is not int or n <= 0 for n in self._partitions + ): + raise ValueError("Logical partitions must have matching rank and positive extents") + if prod(self._partitions) > 2147483647: + raise ValueError("Logical partition exceeds the native lane limit") + self._grid = tuple((n + p - 1) // p for n, p in zip(self._domain, self._partitions, strict=True)) + self._shape = self._grid if kernel.result_cardinality == "block_scalar" else self._domain + + @property + def shape(self): + return self._shape + + @property + def dtype(self): + return self.kernel.output_dtype + + @property + def nbytes(self): + return prod(self.shape) * self.dtype.itemsize + + @property + def partitions(self): + return self._partitions + + @property + def chunks(self): + return tuple(max(n, 1) for n in self.shape) + + @property + def blocks(self): + return self.chunks + + def _values(self, selection=None): + if selection is None: + selection = tuple(slice(None) for _ in self.shape) + elif not isinstance(selection, tuple): + selection = (selection,) + if any( + part is None or isinstance(part, bool) or not isinstance(part, (slice, int, type(Ellipsis))) + for part in selection + ): + return self._values()[selection] + if sum(part is Ellipsis for part in selection) > 1: + raise IndexError("Only one ellipsis is allowed") + if Ellipsis in selection: + position = selection.index(Ellipsis) + selection = ( + selection[:position] + + (slice(None),) * (len(self.shape) - len(selection) + 1) + + selection[position + 1 :] + ) + selection += (slice(None),) * (len(self.shape) - len(selection)) + if len(selection) != len(self.shape): + raise IndexError("Too many indices") + selected, squeeze = [], [] + for axis, (part, size) in enumerate(zip(selection, self.shape, strict=True)): + if isinstance(part, int): + index = part + size if part < 0 else part + if not 0 <= index < size: + raise IndexError("Portable output index out of range") + selected.append(np.array([index], dtype=np.intp)) + squeeze.append(axis) + else: + selected.append(np.arange(*part.indices(size), dtype=np.intp)) + result = np.empty(tuple(len(indices) for indices in selected), dtype=self.dtype) + scalar = self.kernel.result_cardinality == "block_scalar" + groups = [ + np.unique(indices if scalar else indices // p) + for indices, p in zip(selected, self._partitions, strict=True) + ] + # Always read complete original groups. Output slicing/rechunking never + # changes reduction membership or the domain used by native coordinates. + for index in product(*groups): + index = tuple(int(i) for i in index) + origin = tuple(i * p for i, p in zip(index, self._partitions, strict=True)) + extent = tuple( + min(p, n - o) for p, n, o in zip(self._partitions, self._domain, origin, strict=True) + ) + item = tuple(slice(o, o + n) for o, n in zip(origin, extent, strict=True)) + operands = {} + for name, value in self.inputs.items(): + rank = len(value.shape) + source_item = ( + tuple( + slice(0, 1) if size == 1 else part + for size, part in zip(value.shape, item[len(item) - rank :], strict=True) + ) + if rank + else () + ) + operands[name] = np.broadcast_to(np.asarray(value[source_item]), extent) + context = ( + {"logical_shape": self._domain, "block_origin": origin} if self.kernel.context_ndim else {} + ) + block = self.kernel.evaluate_block(operands, block_shape=extent, **context) + positions = [ + np.flatnonzero(indices == i) + if scalar + else np.flatnonzero((indices >= o) & (indices < o + n)) + for indices, i, o, n in zip(selected, index, origin, extent, strict=True) + ] + local = [indices[pos] - o for indices, pos, o in zip(selected, positions, origin, strict=True)] + if not self.shape: + result[...] = block + else: + result[np.ix_(*positions)] = block if scalar else block[np.ix_(*local)] + return np.squeeze(result, axis=tuple(squeeze)) if squeeze else result + + def __getitem__(self, item): + return self._values(item) + + def compute(self, item=None, **kwargs): + if item is not None: + return blosc2.asarray(self._values(item), **kwargs) + output = blosc2.empty(self.shape, dtype=self.dtype, **kwargs) + if not self.shape: + output[()] = self._values() + return output + # Output storage tiles may intersect several immutable source groups. + # Re-evaluation at a tile boundary is permitted, repartitioning is not. + for origin in product(*(range(0, n, c) for n, c in zip(self.shape, output.chunks, strict=True))): + item = tuple( + slice(o, min(o + c, n)) for o, c, n in zip(origin, output.chunks, self.shape, strict=True) + ) + output[item] = self._values(item) + return output + + def rechunk(self, *, chunks=None, blocks=None, **kwargs): + """Materialize into a different storage grid, retaining logical groups.""" + return self.compute(chunks=chunks, blocks=blocks, **kwargs) + + def sort(self, order=None): + raise NotImplementedError("Portable sorting is not part of the kernel contract") + + def argsort(self, order=None): + raise NotImplementedError("Portable sorting is not part of the kernel contract") + + def save(self, urlpath=None, **kwargs): + from blosc2.b2objects import ( + encode_b2object_payload, + make_b2object_carrier, + write_b2object_payload, + write_b2object_user_vlmeta, + ) + + if urlpath is None: + raise ValueError("A portable carrier requires urlpath") + payload = encode_b2object_payload(self) # Preflight every reference before opening destination. + from blosc2.msgpack_utils import msgpack_packb + + msgpack_packb(payload) + msgpack_packb(self._get_user_vlmeta()) + carrier = make_b2object_carrier("portable", self.shape, self.dtype, urlpath=urlpath, **kwargs) + write_b2object_payload(carrier, payload) + write_b2object_user_vlmeta(carrier, self._get_user_vlmeta()) + self.array, self.schunk = carrier, carrier.schunk + + def to_cframe(self, **kwargs): + from blosc2.b2objects import ( + encode_b2object_payload, + make_b2object_carrier, + write_b2object_payload, + write_b2object_user_vlmeta, + ) + + payload = encode_b2object_payload(self) + carrier = make_b2object_carrier("portable", self.shape, self.dtype, **kwargs) + write_b2object_payload(carrier, payload) + write_b2object_user_vlmeta(carrier, self._get_user_vlmeta()) + return carrier.to_cframe() diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index aa058e044..d53232eba 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1976,7 +1976,9 @@ def process_opened_object(res, *, deserialize=DeserializeMode.SAFE): return proxy if "b2o" in meta: - if deserialize is DeserializeMode.SAFE: + if deserialize is DeserializeMode.SAFE and ( + not isinstance(meta["b2o"], dict) or meta["b2o"].get("kind") not in {"portable", "lazyexpr"} + ): marker = meta["b2o"] kind = marker.get("kind", "b2o") if isinstance(marker, dict) else "b2o" raise UnsafeDeserializationError(str(kind)) diff --git a/src/blosc2/tree_store.py b/src/blosc2/tree_store.py index bdc390390..dc46921ba 100644 --- a/src/blosc2/tree_store.py +++ b/src/blosc2/tree_store.py @@ -441,6 +441,9 @@ def __setitem__( """ key = self._validate_key(key) + from blosc2.b2objects import preflight_persistence + + preflight_persistence(value) if isinstance(value, blosc2.RemoteStore): self._set_remote_store_reference(key, value) return diff --git a/tests/ctable/test_ctable_computed_cols.py b/tests/ctable/test_ctable_computed_cols.py index 9c90040ef..f50e22d93 100644 --- a/tests/ctable/test_ctable_computed_cols.py +++ b/tests/ctable/test_ctable_computed_cols.py @@ -876,16 +876,15 @@ def test_dsl_computed_column_no_storage(): def test_dsl_computed_column_persistence(tmp_path): - """DSL kernel source is persisted and the column is available after open.""" + """Authored computed columns persist with an explicit row-domain contract.""" t = _make_invoice_table(5) - t.add_computed_column("total", total, inputs=["price", "qty"]) + t.add_portable_computed_column( + "total", total, inputs={"price": "price", "qty": "qty"}, dtype="float64", row_domain="independent" + ) path = str(tmp_path / "tbl") t.save(path) - - t2 = blosc2.open(path) - assert "total" in t2._computed_cols - assert t2._computed_cols["total"]["kind"] == "dsl" - np.testing.assert_allclose(t2["total"][:], [1.0, 4.0, 9.0, 16.0, 25.0]) + restored = blosc2.CTable.open(path) + np.testing.assert_allclose(restored["total"][:], [1, 4, 9, 16, 25]) def test_dsl_computed_column_inputs_mismatch_raises(): @@ -982,17 +981,16 @@ def test_dsl_generated_column_autofill_append(): def test_dsl_generated_column_persistence_and_append(tmp_path): - """DSL generated column survives save/open and auto-fills after reload.""" + """Authored row transformers persist and refill appended rows natively.""" t = _make_invoice_table(3) - t.add_generated_column("total", values=total, inputs=["price", "qty"]) + t.add_portable_generated_column( + "total", total, inputs={"price": "price", "qty": "qty"}, dtype="float64", row_domain="independent" + ) path = str(tmp_path / "tbl") t.save(path) - - t2 = blosc2.open(path, mode="a") - assert "total" in t2._materialized_cols - assert t2._materialized_cols["total"]["transformer_kind"] == "dsl" - t2.append((4.0, 4, 0.1)) - np.testing.assert_allclose(t2["total"][:], [1.0, 4.0, 9.0, 16.0]) + restored = blosc2.CTable.open(path, mode="a") + restored.append((4.0, 4, 0.1)) + np.testing.assert_allclose(restored["total"][:], [1.0, 4.0, 9.0, 16.0]) def test_dsl_generated_column_refresh(tmp_path): @@ -1025,19 +1023,15 @@ def test_dsl_jit_backend_persisted_in_schema(): @pytest.mark.skipif(sys.platform == "win32", reason="cc backend requires a system C compiler") def test_dsl_jit_backend_restored_after_open(tmp_path): - """jit_backend is restored from schema on open and used for auto-fill.""" + """Selecting a JIT backend cannot bypass portable persistence preflight.""" t = _make_invoice_table(3) t.add_generated_column( "total", values=blosc2.lazyudf(total, (t.price, t.qty), jit_backend="cc"), ) path = str(tmp_path / "tbl") - t.save(path) - - t2 = blosc2.open(path, mode="a") - assert t2._materialized_cols["total"].get("jit_backend") == "cc" - t2.append((4.0, 4, 0.1)) - np.testing.assert_allclose(t2["total"][:], [1.0, 4.0, 9.0, 16.0]) + with pytest.raises(TypeError, match="row-domain"): + t.save(path) def test_dsl_no_jit_backend_not_in_schema(): diff --git a/tests/ctable/test_ctable_dsl_columns.py b/tests/ctable/test_ctable_dsl_columns.py index 6aaf5c2b0..5f514c2b5 100644 --- a/tests/ctable/test_ctable_dsl_columns.py +++ b/tests/ctable/test_ctable_dsl_columns.py @@ -152,23 +152,17 @@ def test_dsl_computed_in_where_streams_multichunk(): def test_dsl_computed_roundtrip(tmp_path): path = str(tmp_path / "dsl.b2d") t = _make_table(urlpath=path, mode="w") - t.add_computed_column("r", k_loop, inputs=["a", "b"]) + with pytest.raises(TypeError, match="portable artifacts"): + t.add_computed_column("r", k_loop, inputs=["a", "b"]) t.close() - # Stored schema carries kind:dsl + dsl_source and no expression. + # Rejection leaves no source-backed computed recipe in the stored schema. meta = blosc2.open(f"{path}/_meta.b2f") sd = json.loads(meta.vlmeta["schema"]) - (cc,) = sd["computed_columns"] - assert cc["kind"] == "dsl" - assert "dsl_source" in cc - assert "expression" not in cc + assert "computed_columns" not in sd t2 = blosc2.open(path) - np.testing.assert_array_equal(np.asarray(t2["r"][:]), np.arange(20) + 20) - a = np.arange(20) - r = a + 20 - sel = t2.where("(r > 25) & (a < 18)")[:] - assert len(sel) == int(((r > 25) & (a < 18)).sum()) + assert "r" not in t2.col_names t2.close() @@ -230,15 +224,14 @@ def test_dsl_generated_create_index(): def test_dsl_generated_roundtrip(tmp_path): path = str(tmp_path / "gen.b2d") t = _make_table(n=10, urlpath=path, mode="w") - t.add_generated_column("g", values=k_add, inputs=["a", "b"], dtype=blosc2.int64()) + with pytest.raises(TypeError, match="portable artifacts"): + t.add_generated_column("g", values=k_add, inputs=["a", "b"], dtype=blosc2.int64()) t.close() meta = blosc2.open(f"{path}/_meta.b2f") sd = json.loads(meta.vlmeta["schema"]) - (m,) = sd["materialized_columns"] - assert m["transformer_kind"] == "dsl" - assert "dsl_source" in m + assert "materialized_columns" not in sd t2 = blosc2.open(path) - np.testing.assert_array_equal(np.asarray(t2["g"][:]), np.arange(10) + 10) + assert "g" not in t2.col_names t2.close() diff --git a/tests/ndarray/test_dsl_kernels.py b/tests/ndarray/test_dsl_kernels.py index 3d6097fe0..b9747b202 100644 --- a/tests/ndarray/test_dsl_kernels.py +++ b/tests/ndarray/test_dsl_kernels.py @@ -920,17 +920,18 @@ def kernel_save_loop(x, y): return acc -def _save_reload_compute(kernel, inputs_np, inputs_b2, dtype, urlpaths, extra_kwargs=None): - """Save a LazyUDF backed by *kernel*, reload it, and return (reloaded_expr, result).""" +def _save_compute(kernel, inputs_b2, dtype, urlpaths, extra_kwargs=None): + """New authoring persists a native recipe, not a reconstructed Python function.""" lazy = blosc2.lazyudf(kernel, inputs_b2, dtype=dtype, **(extra_kwargs or {})) lazy.save(urlpath=urlpaths["lazy"]) - reloaded = blosc2.open(urlpaths["lazy"], mode="r", deserialize="full") - return reloaded, reloaded.compute() + restored = blosc2.open(urlpaths["lazy"]) + assert restored.partitions == lazy.blocks + assert restored.kernel.schema_version == "1.0" + return restored, restored.compute() def test_dsl_save_simple(tmp_path): - """Simple quadratic kernel: dsl_source and DSLKernel type survive a round-trip.""" - from blosc2.dsl_kernel import DSLKernel + """A quadratic author saves and safely reloads as native portable code.""" shape = (16, 16) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -939,20 +940,15 @@ def test_dsl_save_simple(tmp_path): b = blosc2.asarray(nb, urlpath=str(tmp_path / "b.b2nd"), mode="w") urlpaths = {"lazy": str(tmp_path / "lazy.b2nd")} - reloaded, result = _save_reload_compute(kernel_save_simple, (na, nb), (a, b), np.float64, urlpaths) - - assert isinstance(reloaded, blosc2.LazyUDF) - assert isinstance(reloaded.func, DSLKernel), "func must be a DSLKernel after reload" - assert reloaded.func.dsl_source is not None, "dsl_source must be preserved" - assert "kernel_save_simple" in reloaded.func.dsl_source + reloaded, result = _save_compute(kernel_save_simple, (a, b), np.float64, urlpaths) + assert isinstance(reloaded.kernel, blosc2.PortableKernel) expected = (na + nb) ** 2 # (x+y)^2 == x^2 + y^2 + 2xy np.testing.assert_allclose(result[...], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_clamp(tmp_path): - """Kernel with a `where` call survives save/reload and produces correct values.""" - from blosc2.dsl_kernel import DSLKernel + """Normalized where authoring roundtrips through native persistence.""" shape = (20, 20) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -961,18 +957,15 @@ def test_dsl_save_clamp(tmp_path): b = blosc2.asarray(nb, urlpath=str(tmp_path / "b.b2nd"), mode="w") urlpaths = {"lazy": str(tmp_path / "lazy.b2nd")} - reloaded, result = _save_reload_compute(kernel_save_clamp, (na, nb), (a, b), np.float64, urlpaths) - - assert isinstance(reloaded.func, DSLKernel) - assert reloaded.func.dsl_source is not None + reloaded, result = _save_compute(kernel_save_clamp, (a, b), np.float64, urlpaths) + assert isinstance(reloaded.kernel, blosc2.PortableKernel) expected = np.where(na + nb > 1.5, na + nb, 1.5) np.testing.assert_allclose(result[...], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_loop(tmp_path): - """Kernel with a loop (full DSL function) survives save/reload.""" - from blosc2.dsl_kernel import DSLKernel + """A static loop author roundtrips through native persistence.""" shape = (12, 12) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -981,19 +974,15 @@ def test_dsl_save_loop(tmp_path): b = blosc2.asarray(nb, urlpath=str(tmp_path / "b.b2nd"), mode="w") urlpaths = {"lazy": str(tmp_path / "lazy.b2nd")} - reloaded, result = _save_reload_compute(kernel_save_loop, (na, nb), (a, b), np.float64, urlpaths) - - assert isinstance(reloaded.func, DSLKernel) - assert reloaded.func.dsl_source is not None - assert "for i in range(3):" in reloaded.func.dsl_source + reloaded, result = _save_compute(kernel_save_loop, (a, b), np.float64, urlpaths) + assert isinstance(reloaded.kernel, blosc2.PortableKernel) expected = kernel_save_loop.func(na, nb) np.testing.assert_allclose(result[...], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_getitem(tmp_path): - """Reloaded DSL kernel supports __getitem__ (sliced access), not just compute().""" - from blosc2.dsl_kernel import DSLKernel + """Reloaded native recipes retain sliced access.""" shape = (16, 16) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -1003,16 +992,13 @@ def test_dsl_save_getitem(tmp_path): lazy = blosc2.lazyudf(kernel_save_simple, (a, b), dtype=np.float64) lazy.save(urlpath=str(tmp_path / "lazy.b2nd")) - reloaded = blosc2.open(str(tmp_path / "lazy.b2nd"), mode="r", deserialize="full") - - assert isinstance(reloaded.func, DSLKernel) + reloaded = blosc2.open(tmp_path / "lazy.b2nd") expected = (na + nb) ** 2 np.testing.assert_allclose(reloaded[()], expected, rtol=1e-5, atol=1e-6) def test_dsl_save_input_names_match(tmp_path): - """After reload, input_names in the DSLKernel match the original kernel.""" - from blosc2.dsl_kernel import DSLKernel + """Native recipes preserve named positional bindings.""" shape = (10, 10) na = np.linspace(0, 1, np.prod(shape), dtype=np.float32).reshape(shape) @@ -1022,11 +1008,20 @@ def test_dsl_save_input_names_match(tmp_path): lazy = blosc2.lazyudf(kernel_save_simple, (a, b), dtype=np.float64) lazy.save(urlpath=str(tmp_path / "lazy.b2nd")) - reloaded = blosc2.open(str(tmp_path / "lazy.b2nd"), mode="r", deserialize="full") - - assert isinstance(reloaded.func, DSLKernel) - assert reloaded.func.input_names == ["x", "y"] - assert list(reloaded.inputs_dict.keys()) == ["x", "y"] + reloaded = blosc2.open(tmp_path / "lazy.b2nd") + assert list(reloaded.kernel.input_dtypes) == ["x", "y"] + assert list(reloaded.inputs) == ["x", "y"] + grouped = blosc2.asarray( + np.arange(8, dtype="int64"), chunks=(8,), blocks=(2,), urlpath=tmp_path / "grouped.b2nd", mode="w" + ) + kernel = blosc2.DSLKernel.from_source("def grouped(x):\n return x + sum(x) + _i0\n") + authored = blosc2.lazyudf(kernel, (grouped,), dtype="int64") + frame = authored.to_cframe() + restored = blosc2.from_cframe(frame) + expected = np.arange(8) * 2 + np.repeat(np.arange(8).reshape(4, 2).sum(axis=1), 2) + assert restored.partitions == (2,) + np.testing.assert_array_equal(restored[:], expected) + np.testing.assert_array_equal(restored[1:7:2], expected[1:7:2]) def test_dsl_save_dictstore_operands(tmp_path): @@ -1047,16 +1042,7 @@ def test_dsl_save_dictstore_operands(tmp_path): b = dstore["/b"] lazy = blosc2.lazyudf(kernel_save_simple, (a, b), dtype=np.float64) lazy.save(urlpath=str(expr_path)) - - carrier = blosc2.open(str(expr_path), mode="r", deserialize="full").array - assert carrier.schunk.vlmeta["b2o"]["operands"] == { - "o0": {"kind": "dictstore_key", "version": 1, "urlpath": "ops.b2z", "key": "/a"}, - "o1": {"kind": "dictstore_key", "version": 1, "urlpath": "ops.b2z", "key": "/b"}, - } - - reloaded = blosc2.open(str(expr_path), mode="r", deserialize="full") - expected = (np.arange(shape[0], dtype=np.float64) * 3) ** 2 - np.testing.assert_allclose(reloaded.compute()[...], expected, rtol=1e-5, atol=1e-6) + np.testing.assert_array_equal(blosc2.open(expr_path)[:], (np.arange(10) * 3) ** 2) # --- plans/dsl-glitches.md regressions: G1 (one-per-line), G2 (input reassign), diff --git a/tests/ndarray/test_lazyudf.py b/tests/ndarray/test_lazyudf.py index b75f292d0..39811f9ad 100644 --- a/tests/ndarray/test_lazyudf.py +++ b/tests/ndarray/test_lazyudf.py @@ -483,9 +483,8 @@ def test_save_ludf(): expr = blosc2.lazyudf(udf1p, (array,), np.float64, cparams=cparams) - expr.save(urlpath=urlpath) - del expr - expr = blosc2.open(urlpath, mode="r", deserialize="full") + with pytest.raises(TypeError, match="PortableKernel"): + expr.save(urlpath=urlpath) assert isinstance(expr, blosc2.LazyUDF) res_lazyexpr = expr.compute() np.testing.assert_array_equal(res_lazyexpr[:], npc) @@ -493,9 +492,8 @@ def test_save_ludf(): if blosc2._HAS_NUMBA: expr = blosc2.lazyudf(udf1p_numba, (array,), np.float64) - expr.save(urlpath=urlpath) - del expr - expr = blosc2.open(urlpath, mode="r", deserialize="full") + with pytest.raises(TypeError, match="PortableKernel"): + expr.save(urlpath=urlpath) assert isinstance(expr, blosc2.LazyUDF) res_lazyexpr = expr.compute() np.testing.assert_array_equal(res_lazyexpr[:], npc) @@ -511,18 +509,11 @@ def test_lazyudf_vlmeta_roundtrip(tmp_path): expr.vlmeta["name"] = "increment" expr.vlmeta["attrs"] = {"version": 1} - expr.save(urlpath=str(expr_path)) - - restored = blosc2.open(str(expr_path), mode="r", deserialize="full") - - assert isinstance(restored, blosc2.LazyUDF) - assert restored.vlmeta["name"] == "increment" - assert restored.vlmeta["attrs"] == {"version": 1} - - with blosc2.open(str(expr_path), mode="r", deserialize="full") as restored_ctx: - assert isinstance(restored_ctx, blosc2.LazyUDF) - assert restored_ctx.vlmeta["name"] == "increment" - assert restored_ctx.vlmeta["attrs"] == {"version": 1} + with pytest.raises(TypeError, match="PortableKernel"): + expr.save(urlpath=str(expr_path)) + assert not expr_path.exists() + assert expr.vlmeta["name"] == "increment" + assert expr.vlmeta["attrs"] == {"version": 1} # Test get_chunk method diff --git a/tests/test_b2objects.py b/tests/test_b2objects.py index 9eb0cde16..921d5246a 100644 --- a/tests/test_b2objects.py +++ b/tests/test_b2objects.py @@ -10,6 +10,7 @@ from pathlib import Path import numpy as np +import pytest import blosc2 import blosc2.c2array as blosc2_c2array @@ -148,27 +149,25 @@ def test_lazyudf_from_cframe_roundtrip(tmp_path): a = blosc2.asarray(np.arange(5, dtype=np.float64), urlpath=tmp_path / "a.b2nd", mode="w") b = blosc2.asarray(np.arange(5, dtype=np.float64) * 2, urlpath=tmp_path / "b.b2nd", mode="w") expr = blosc2.lazyudf(kernel_add_square, (a, b), dtype=np.float64) - carrier = blosc2.ndarray_from_cframe(expr.to_cframe()) + restored = blosc2.from_cframe(expr.to_cframe()) + np.testing.assert_allclose(restored[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + from blosc2.msgpack_utils import msgpack_packb, msgpack_unpackb - assert carrier.schunk.meta["b2o"] == {"kind": "lazyudf", "version": 1} - payload = carrier.schunk.vlmeta["b2o"] - assert payload["kind"] == "lazyudf" - assert payload["version"] == 1 - assert payload["function_kind"] == "dsl" - assert payload["dsl_version"] == 1 - assert payload["name"] == "kernel_add_square" - assert "kernel_add_square" in payload["udf_source"] - assert payload["dtype"] == np.dtype(np.float64).str - assert payload["shape"] == [5] - assert payload["operands"] == { - "o0": {"kind": "urlpath", "version": 1, "urlpath": (tmp_path / "a.b2nd").as_posix()}, - "o1": {"kind": "urlpath", "version": 1, "urlpath": (tmp_path / "b.b2nd").as_posix()}, - } + offset = np.float32(2) - restored = blosc2.from_cframe(expr.to_cframe(), deserialize="full") + @blosc2.dsl_kernel + def captured(x, scale): + return x * scale + offset - assert isinstance(restored, blosc2.LazyUDF) - np.testing.assert_allclose(restored[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + authored = blosc2.lazyudf(captured, (a, np.int16(3)), dtype="float64") + authored.vlmeta["label"] = "captured" + packed = msgpack_packb([authored]) + framed = authored.to_cframe() + offset = np.float32(99) + np.testing.assert_array_equal(msgpack_unpackb(packed)[0][:], np.arange(5) * 3 + 2) + framed_recipe = blosc2.from_cframe(framed) + np.testing.assert_array_equal(framed_recipe[:], np.arange(5) * 3 + 2) + assert framed_recipe.vlmeta["label"] == "captured" def test_lazyudf_open_roundtrip(tmp_path): @@ -178,10 +177,27 @@ def test_lazyudf_open_roundtrip(tmp_path): urlpath = tmp_path / "expr.b2nd" expr.save(urlpath) - restored = blosc2.open(urlpath, mode="r", deserialize="full") - - assert isinstance(restored, blosc2.LazyUDF) + restored = blosc2.open(urlpath, mode="r") np.testing.assert_allclose(restored[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + nested = expr + 1 + nested.save(tmp_path / "nested.b2nd") + np.testing.assert_array_equal(blosc2.open(tmp_path / "nested.b2nd")[:], (np.arange(5) * 3) ** 2 + 1) + for body, expected in ( + ("print(x)\n return x", "print"), + ("return callback(x)", "invalid_source"), + ("return sum(x)", "block scalar"), + ): + invalid = blosc2.DSLKernel.from_source(f"def invalid(x):\n {body}\n") + bad = blosc2.lazyudf(invalid, (a,), dtype="float64") + before = urlpath.read_bytes() + with pytest.raises((blosc2.PortableArtifactError, ValueError), match=expected): + bad.save(urlpath, mode="w") + assert urlpath.read_bytes() == before + fixture = Path(__file__).parent / "data" / "legacy_lazyudf_v1" / "expr.b2nd" + legacy = blosc2.open(fixture, deserialize="full") + with pytest.raises(TypeError, match="legacy"): + legacy.save(urlpath, mode="w") + assert urlpath.read_bytes() == before def test_b2z_bundle_with_lazy_recipes_opens_read_only(tmp_path): @@ -197,13 +213,11 @@ def test_b2z_bundle_with_lazy_recipes_opens_read_only(tmp_path): udf = blosc2.lazyudf(kernel_add_square, (a, b), dtype=np.float64, shape=a.shape) store["/recipes/expr"] = blosc2.ndarray_from_cframe(expr.to_cframe()) - store["/recipes/udf"] = blosc2.ndarray_from_cframe(udf.to_cframe()) + store["/recipes/udf"] = udf with blosc2.open(str(bundle_path), mode="r", deserialize="full") as store: restored_expr = store["/recipes/expr"] - restored_udf = store["/recipes/udf"] assert isinstance(restored_expr, blosc2.LazyExpr) - assert isinstance(restored_udf, blosc2.LazyUDF) np.testing.assert_allclose(restored_expr.compute()[:], np.arange(5, dtype=np.float64) * 3) - np.testing.assert_allclose(restored_udf.compute()[:], (np.arange(5, dtype=np.float64) * 3) ** 2) + np.testing.assert_allclose(store["/recipes/udf"][:], (np.arange(5, dtype=np.float64) * 3) ** 2) diff --git a/tests/test_deserialization.py b/tests/test_deserialization.py index 1aae429dd..38b9a5d56 100644 --- a/tests/test_deserialization.py +++ b/tests/test_deserialization.py @@ -180,8 +180,7 @@ def test_active_top_level_object_requires_full(tmp_path): path = tmp_path / "expression.b2nd" expression.save(path) - with pytest.raises(blosc2.UnsafeDeserializationError, match="lazyexpr"): - blosc2.open(path) + np.testing.assert_array_equal(blosc2.open(path)[:], np.arange(4) + 1) trusted = blosc2.open(path, deserialize="full") np.testing.assert_array_equal(trusted[:], np.arange(4) + 1) diff --git a/tests/test_dsl_portable.py b/tests/test_dsl_portable.py index 474f8a9a8..8db23ddf7 100644 --- a/tests/test_dsl_portable.py +++ b/tests/test_dsl_portable.py @@ -16,17 +16,13 @@ # Native CTest runs every fixture under its supported execution engines. Python -# tests only distinct host boundaries here, plus every exact output conversion; +# tests only distinct host boundaries here and representative output conversions; # numerical/JIT scenarios are covered separately by test_portable_artifact.py. CASES = [ "affine", - "chains", - "loops", "special_floats", - "flow", "math", "division_bool", - "division_literals", "division_locals", "while_cap_exceeded", "unknown_name", @@ -36,12 +32,16 @@ "missing_return_loop_ok", ] TYPES = ("bool", "int32", "int64", "float32", "float64") -EXCLUDED = ( - set((CORPUS / "frozen-excluded.txt").read_text().split()) - if CORPUS is not None and (CORPUS / "frozen-excluded.txt").is_file() - else set() -) -CASES += [f"convert_{input_type}_{output_type}" for input_type in TYPES for output_type in TYPES] +CASES += [ + f"convert_{input_type}_{output_type}" + for input_type, output_type in ( + ("bool", "int64"), + ("int32", "bool"), + ("int64", "float32"), + ("float32", "float64"), + ("float64", "int32"), + ) +] def corpus_source(case): @@ -67,34 +67,37 @@ def assert_typed_result(actual, expected, *, exact=False): @pytest.mark.parametrize("case", CASES) def test_native_source_corpus(case, portable_validator, monkeypatch): - jit = True if case.startswith("while_cap_"): monkeypatch.setenv("ME_DSL_WHILE_MAX_ITERS", "3") if CORPUS is None: pytest.skip("Set MINIEXPR_PORTABLE_CORPUS to the native miniexpr fixture directory") - # An explicit corpus is required to match the pinned profile. Do not - # silently skip missing files or misinterpret an older exclusion manifest. + # An explicit corpus must match draft 1.0; never silently skip missing files. source = corpus_source(case).read_text() words = (CORPUS / f"{case}.txt").read_text().split() outcome, input_type, output_type = words[:3] count, nvars = map(int, words[3:5]) names = words[5 : 5 + nvars] info = portable_validator(source, dict.fromkeys(names, input_type), output_type) - assert info["valid"] == (outcome != "compile_error" and case not in EXCLUDED), info - if case in EXCLUDED: - assert info["status"] == "unsupported_feature" + assert info["valid"] == (outcome != "compile_error"), info rows = np.array(words[5 + nvars :]).reshape(count, nvars + 1) input_dtype, output_dtype = np.dtype(input_type), np.dtype(output_type) expected = typed_values(rows[:, -1], output_dtype) kernel = DSLKernel.from_source(source) by_name = {name: typed_values(rows[:, index], input_dtype) for index, name in enumerate(names)} - inputs = [by_name[name] for name in kernel.input_names] if outcome in {"ok", "ok_exact"}: - result = blosc2.lazyudf(kernel, inputs, dtype=output_dtype, jit=jit)[:] + artifact = kernel.export(dict.fromkeys(names, input_type), output_type) + loaded = blosc2.PortableKernel.from_json(artifact, jit=True) + assert not loaded.has_jit + result = loaded.evaluate(by_name) assert_typed_result(result, expected, exact=outcome == "ok_exact") + elif outcome == "compile_error": + with pytest.raises(blosc2.PortableArtifactError): + kernel.export(dict.fromkeys(names, input_type), output_type) else: - with pytest.raises((NotImplementedError, RuntimeError)): - blosc2.lazyudf(kernel, inputs, dtype=output_dtype, jit=jit)[:] + artifact = kernel.export(dict.fromkeys(names, input_type), output_type) + loaded = blosc2.PortableKernel.from_json(artifact, jit=True) + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + loaded.evaluate(by_name) assert kernel.dsl_source == source assert kernel.func is None @@ -105,8 +108,7 @@ def test_native_source_corpus(case, portable_validator, monkeypatch): runner, str(corpus_source(case)), str(CORPUS / f"{case}.txt"), - "on" if jit else "off", - *(["native"] if case in EXCLUDED else []), + "on", ], check=True, capture_output=True, @@ -116,7 +118,7 @@ def test_native_source_corpus(case, portable_validator, monkeypatch): if outcome == "compile_error": assert lines == [outcome] else: - assert lines[0] == f"jit={int(jit)}" + assert lines[0] == "jit=0" if outcome == "eval_error": assert lines[1:] == [outcome] else: @@ -177,14 +179,14 @@ def portable_validator(): @pytest.mark.parametrize( ("body", "status"), [ - ("return sum(x)", "unsupported_feature"), - ("return np.sin(x)", "unsupported_feature"), - ("return x.lower()", "unsupported_feature"), - ("return x[0]", "unsupported_feature"), - ("return _flat_idx + x", "unsupported_feature"), - ("return callback(x)", "unsupported_feature"), + ("print(x); return x", "invalid_source"), + ("return np.sin(x)", "invalid_source"), + ("return x.lower()", "invalid_source"), + ("return x[0]", "invalid_source"), + ("return _flat_idx + x", "invalid_source"), + ("return callback(x)", "invalid_source"), ("return CAPTURE + x", "invalid_source"), - ("return sin()", "unsupported_feature"), + ("return sin()", "invalid_source"), ], ) def test_portable_profile_rejections(portable_validator, body, status): @@ -192,8 +194,12 @@ def test_portable_profile_rejections(portable_validator, body, status): assert not info["valid"] assert info["status"] == status assert info["error"] - assert info["line"] == 2 - assert info["column"] > 0 + if "_flat_idx" in body: + # Descriptor-wide missing-context diagnostic is not a source location. + assert info["line"] == 0 + else: + assert info["line"] == 2 + assert info["column"] > 0 def test_portable_profile_signature_and_version(portable_validator): @@ -204,8 +210,10 @@ def test_portable_profile_signature_and_version(portable_validator): "unsupported_version" ) assert portable_validator(source, {"x": "float64"}, "float64")["status"] == "invalid_signature" - assert portable_validator(source, {"x": "float64", "y": "int32"}, "float64")["status"] == ( - "unsupported_feature" + assert portable_validator(source, {"x": "float64", "y": "int32"}, "float64")["valid"] + assert ( + portable_validator(source, inputs, "float64", language_version="0.1")["status"] + == "unsupported_version" ) assert portable_validator(source, inputs, "complex128")["status"] == "unsupported_feature" @@ -216,6 +224,7 @@ def test_portable_profile_never_executes_or_compiles_jit(portable_validator, mon monkeypatch.setenv("CC", "/no/compiler") monkeypatch.setenv("ME_DSL_JIT_COMPILER", "cc") monkeypatch.setenv("ME_DSL_JIT", "1") + monkeypatch.setenv("ME_DSL_FP_MODE", "fast") monkeypatch.setenv("ME_DSL_JIT_CACHE_DIR", str(cache)) # This would reach the iteration cap if executed. Validation must still succeed. source = "# me:compiler=cc\ndef k(x):\n while 1:\n pass\n return x\n" @@ -238,9 +247,9 @@ def test_portable_profile_missing_native_support(monkeypatch): @pytest.mark.parametrize( ("source", "inputs", "version"), [ - ("def k(x):\n return x\n\x00", {"x": "float64"}, "0.1"), - ("def k(x):\n return x\n", {"x\x00": "float64"}, "0.1"), - ("def k(x):\n return x\n", {"x": "float64"}, "0.1\x00"), + ("def k(x):\n return x\n\x00", {"x": "float64"}, "1.0"), + ("def k(x):\n return x\n", {"x\x00": "float64"}, "1.0"), + ("def k(x):\n return x\n", {"x": "float64"}, "1.0\x00"), ], ) def test_portable_profile_rejects_nul(source, inputs, version): @@ -251,6 +260,11 @@ def test_portable_profile_rejects_nul(source, inputs, version): @pytest.mark.parametrize("literal", ["9007199254740993", "9_007_199_254_740_993", "0x20000000000001"]) def test_portable_integer_literal_bound_is_exact(portable_validator, literal): info = portable_validator(f"def k(x):\n return x + {literal}\n", {"x": "int64"}, "int64") - assert not info["valid"] - assert info["status"] == "unsupported_feature" - assert "portable range" in info["error"] + assert info["valid"], info + author = DSLKernel.from_source(f"def k(x):\n return x + {literal}\n") + kernel = blosc2.PortableKernel.from_json(author.export({"x": "int64"}, "int64")) + np.testing.assert_array_equal( + kernel.evaluate({"x": np.array([0, 1], dtype="int64")}), [2**53 + 1, 2**53 + 2] + ) + with pytest.raises(blosc2.PortableArtifactError, match="evaluation_error"): + kernel.evaluate({"x": np.array([np.iinfo("int64").max], dtype="int64")}) diff --git a/tests/test_objectarray.py b/tests/test_objectarray.py index 53c526c59..616746e80 100644 --- a/tests/test_objectarray.py +++ b/tests/test_objectarray.py @@ -267,9 +267,8 @@ def test_msgpack_supports_lazyudf_dslkernel(tmp_path): oarr = blosc2.ObjectArray() oarr.append(udf) restored = oarr[0] - - assert isinstance(restored, blosc2.LazyUDF) - np.testing.assert_allclose(restored[:], expected) + np.testing.assert_array_equal(restored[:], expected) + assert restored.kernel.schema_version == "1.0" def test_msgpack_rejects_in_memory_lazyexpr(): @@ -284,7 +283,7 @@ def test_msgpack_rejects_plain_python_lazyudf(tmp_path): udf = _make_persistent_python_lazyudf(tmp_path) oarr = blosc2.ObjectArray() - with pytest.raises(TypeError, match="DSLKernel"): + with pytest.raises(TypeError, match="PortableKernel"): oarr.append(udf) diff --git a/tests/test_portable_artifact.py b/tests/test_portable_artifact.py index 7e40c22cd..2df5b85f3 100644 --- a/tests/test_portable_artifact.py +++ b/tests/test_portable_artifact.py @@ -26,56 +26,38 @@ def assert_execution_mode(kernel, jit): - # Portable 0.1 certifies Windows' interpreter baseline, not its optional - # accelerator. Still execute all semantic assertions there, including - # JIT-requested fallback. On certified JIT hosts fallback must not hide a - # code-generation regression; JIT_OFF must disable it on every platform. - if jit and sys.platform == "win32": - return - assert kernel.has_jit == jit - - -def assert_frozen_rejection(source, inputs, output): - """Check both authoring and native import, without disguising setup errors.""" - info = blosc2.validate_portable_dsl(source, inputs, output) - assert not info["valid"], info - assert info["status"] == "unsupported_feature", info - with pytest.raises(blosc2.PortableArtifactError) as error: - blosc2.DSLKernel.from_source(source).export(inputs, output) - assert error.value.status == "unsupported_feature" - # Use a valid envelope, then replace only its source. Import must independently - # enforce the native frozen boundary even when authoring is bypassed. - names = ", ".join(inputs) - identity = f"def k({names}):\n return {next(iter(inputs))}\n" - envelope = json.loads(blosc2.DSLKernel.from_source(identity).export(inputs, output)) - envelope["source"] = source - envelope["entry_point"] = ast.parse(source).body[0].name - with pytest.raises(blosc2.PortableArtifactError) as error: - blosc2.PortableKernel.from_json(json.dumps(envelope), jit=False) - assert error.value.status == "unsupported_requirement" + # Draft 1.0 is interpreter-first, including optional JIT requests. + assert kernel.schema_version == "1.0" + assert not kernel.has_jit @pytest.mark.parametrize( - ("expression", "input_dtype", "output_dtype"), + ("expression", "input_dtype", "output_dtype", "expected"), [ - ("sin(x)", "float64", "float64"), - ("cos(x)", "float32", "float32"), - ("int(x + 0.25)", "float64", "int64"), - ("float(float(x))", "float32", "float32"), - ("bool((x + 1.0) - x)", "float32", "bool"), - ("(x + 1.0) > x", "float32", "bool"), - ("(x / 2) > 0", "float64", "bool"), - ("x + 0.5", "int64", "int64"), - ("x == 9007199254740993.0", "int64", "bool"), - ("float(x)", "int64", "float32"), - ("float(-(x * 0.1))", "float32", "float32"), - ("bool(int(x + 0.25))", "float64", "bool"), - ("float(int(x) / 2)", "float32", "float32"), - ("int(x) / 2", "float64", "int64"), + ("sin(x)", "float64", "float64", np.sin([-2.0, 0.0, 2.0])), + ("cos(x)", "float32", "float32", np.cos(np.array([-2, 0, 2], dtype="float32"))), + ("int(x + 0.25)", "float64", "int64", [-1, 0, 2]), + ("float(float(x))", "float32", "float32", [-2, 0, 2]), + ("bool((x + 1.0) - x)", "float32", "bool", [True, True, True]), + ("(x + 1.0) > x", "float32", "bool", [True, True, True]), + ("(x / 2) > 0", "float64", "bool", [False, False, True]), + ("x + 0.5", "int64", "int64", [-1, 0, 2]), + ("x == 9007199254740993.0", "int64", "bool", [False, False, False]), + ("float(x)", "int64", "float32", [-2, 0, 2]), + ("float(-(x * 0.1))", "float32", "float32", [0.2, -0.0, -0.2]), + ("bool(int(x + 0.25))", "float64", "bool", [True, False, True]), + ("float(int(x) / 2)", "float32", "float32", [-1, 0, 1]), + ("int(x) / 2", "float64", "int64", [-1, 0, 1]), ], ) -def test_frozen_boundary_without_corpus(native_artifacts, expression, input_dtype, output_dtype): - assert_frozen_rejection(f"def k(x):\n return {expression}\n", {"x": input_dtype}, output_dtype) +def test_typed_arithmetic_without_corpus(native_artifacts, expression, input_dtype, output_dtype, expected): + source = f"def k(x):\n return {expression}\n" + assert blosc2.validate_portable_dsl(source, {"x": input_dtype}, output_dtype)["valid"] + artifact = blosc2.DSLKernel.from_source(source).export({"x": input_dtype}, output_dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=True) + assert_execution_mode(kernel, True) + actual = kernel.evaluate({"x": np.array([-2, 0, 2], dtype=input_dtype)}) + np.testing.assert_allclose(actual, np.array(expected, dtype=output_dtype), rtol=1e-6) @blosc2.dsl_kernel @@ -120,9 +102,9 @@ def test_capture_closure_and_frontend_sugar(native_artifacts): def trig(x): return np.sin(x) * scale - with pytest.raises(blosc2.PortableArtifactError) as error: - trig.export({"x": "float32"}, "float32") - assert error.value.status == "unsupported_feature" + kernel = blosc2.PortableKernel.from_json(trig.export({"x": "float32"}, "float32"), jit=False) + values = np.array([0, 1, 2], dtype="float32") + np.testing.assert_allclose(kernel.evaluate({"x": values}), np.sin(values) * scale, rtol=1e-6) def test_capture_collision_and_comments(native_artifacts): @@ -180,8 +162,8 @@ def test_explicit_capture_dtype(native_artifacts): def k(x): return x * integer - with pytest.raises(blosc2.PortableArtifactError, match="mixed input dtypes"): - k.export({"x": "float64"}, "float64") + inferred = json.loads(k.export({"x": "float64"}, "float64")) + assert inferred["constants"][0]["dtype"] == "int64" artifact = k.export({"x": "float64"}, "float64", capture_dtypes={"integer": "float64"}) np.testing.assert_array_equal( blosc2.PortableKernel.from_json(artifact, jit=False).evaluate({"x": np.array([3.0])}), [6.0] @@ -191,14 +173,29 @@ def k(x): @pytest.mark.parametrize("value", [np.array(2.0), np.arange(2), object(), "hello"]) -def test_unsupported_captures(native_artifacts, value): +def test_capture_value_types(native_artifacts, value): capture = value + if type(value) is str: + + @blosc2.dsl_kernel + def string_snapshot(x): + return capture + + artifact = string_snapshot.export({"x": "float64"}, "U5") + constant = json.loads(artifact)["constants"][0] + assert constant["dtype"] == "unicode32" + assert constant["encoding"] == "unicode32be-hex" + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(artifact).evaluate({"x": np.zeros(2)}), ["hello", "hello"] + ) + return + @blosc2.dsl_kernel def k(x): return x + capture - with pytest.raises(blosc2.PortableArtifactError, match="Capture 'capture'"): + with pytest.raises(blosc2.PortableArtifactError, match="Captures"): k.export({"x": "float64"}, "float64") @@ -216,7 +213,7 @@ def item(self): def k(x): return x + capture - with pytest.raises(blosc2.PortableArtifactError, match="Capture 'capture'"): + with pytest.raises(blosc2.PortableArtifactError, match="Captures"): k.export({"x": "float64"}, "float64", capture_dtypes={"capture": "float64"}) def callback(x): @@ -247,10 +244,7 @@ def test_artifact_backend_and_repeated_calls(native_artifacts, jit): artifact = affine.export({"x": "float64"}, "float64") kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) - if jit is True and not kernel.has_jit: - pytest.skip("TCC backend not prepared on this host") - if jit is False: - assert not kernel.has_jit + assert_execution_mode(kernel, jit) arrays = [np.arange(600, dtype=np.float64) + i for i in range(4)] if blosc2.IS_WASM: # Pyodide does not provide Python threads. Keep backend selection and @@ -265,9 +259,12 @@ def test_artifact_backend_and_repeated_calls(native_artifacts, jit): def test_capture_range_checks(native_artifacts): kernel = blosc2.DSLKernel.from_source("def k(c):\n return c\n") - for value, dtype in [(2**63, "int64"), (2**31, "int32"), (2**53 + 1, "float64"), (1e100, "float32")]: + for value, dtype in [(2**63, "int64"), (2**31, "int32"), (1e100, "float32")]: with pytest.raises(blosc2.PortableArtifactError): kernel.export({}, dtype, constants={"c": value}, capture_dtypes={"c": dtype}) + # Explicit float conversion rounds to nearest, rather than rejecting precision loss. + rounded = kernel.export({}, "float64", constants={"c": 2**53 + 1}, capture_dtypes={"c": "float64"}) + assert blosc2.PortableKernel.from_json(rounded).evaluate({}, shape=())[()] == float(2**53) def test_nan_payload_and_explicit_float_narrowing(native_artifacts): @@ -366,10 +363,24 @@ def test_export_runs_in_standalone_c(native_artifacts, tmp_path): def test_native_diagnostics_and_runtime_errors(native_artifacts): artifact = affine.export({"x": "float64"}, "float64") - changed = artifact.replace('"schema_version":"0.1"', '"schema_version":"99"') + changed = artifact.replace('"schema_version":"1.0"', '"schema_version":"99"') with pytest.raises(blosc2.PortableArtifactError) as error: blosc2.PortableKernel.from_json(changed, jit=False) assert error.value.status == "unsupported_requirement" + for field in ('"schema_version":"1.0"', '"version":"1.0"'): + with pytest.raises(blosc2.PortableArtifactError, match=r"unsupported.*version"): + blosc2.PortableKernel.from_json(artifact.replace(field, field.replace("1.0", "0.1"))) + retired_manifest = json.loads(artifact) + retired_manifest["schema_version"] = retired_manifest["language"]["version"] = "0.1" + retired_manifest["requires"] = ["core-scalar"] + retired_manifest["output"]["contract"] = "scalar-per-element" + retired_manifest.pop("context") + # Reject the actual retired envelope before applying 1.0's required fields. + with pytest.raises(blosc2.PortableArtifactError, match=r"unsupported.*version"): + blosc2.PortableKernel.from_json(json.dumps(retired_manifest)) + with pytest.raises(blosc2.PortableArtifactError) as retired: + affine.export({"x": "float64"}, "float64", version="0.1") + assert retired.value.status == "unsupported_version" changed = artifact.replace('"schema_version":', '"schema_version":"0.1","schema_version":') with pytest.raises(blosc2.PortableArtifactError) as error: blosc2.PortableKernel.from_json(changed, jit=False) @@ -424,9 +435,12 @@ def test_numeric_execution_shapes(native_artifacts, compiler, jit, count): "float32", " rounded = float(x) + 0.1\n return rounded - float(x)\n", [2**24, -(2**24)], - [0.0, 0.0], + [ + np.float32((float(2**24) + 0.1) - float(2**24)), + np.float32((float(-(2**24)) + 0.1) - float(-(2**24))), + ], ), - ("float64", "float64", " truth = bool(x)\n return -truth\n", [0.0, 1.0], [-0.0, -1.0]), + ("float64", "float64", " truth = bool(x)\n return -truth\n", [0.0, 1.0], [0.0, -1.0]), ] for input_dtype, output_dtype, body, samples, expected in cases: source = f"# me:compiler={compiler}\ndef k(x):\n{body}" @@ -461,9 +475,6 @@ def test_numeric_execution_shapes(native_artifacts, compiler, jit, count): ) def test_bool_output_numeric_arithmetic(native_artifacts, body, samples): compiler, jit, count = "tcc", False, 257 - if "int(x +" in body: - assert_frozen_rejection(f"# me:compiler={compiler}\ndef k(x):\n{body}", {"x": "bool"}, "bool") - return artifact = blosc2.DSLKernel.from_source(f"# me:compiler={compiler}\ndef k(x):\n{body}").export( {"x": "bool"}, "bool" ) @@ -522,13 +533,13 @@ def test_bool_cast_floating_arithmetic( elif input_dtype.startswith("float"): samples += [-0.0, np.finfo(input_dtype).smallest_subnormal, np.inf, -np.inf, np.nan] values = np.resize(np.array(samples, dtype=input_dtype), count) - truth = (values != 0).astype(output_dtype) + truth = (values != 0).astype("int64") expected = { "sum": truth + truth, "difference": truth - truth, "product": truth * truth, "negation": -truth, - }[operation] + }[operation].astype(output_dtype) for _ in range(2): actual = kernel.evaluate({"x": values}) np.testing.assert_array_equal(actual, expected) @@ -559,10 +570,10 @@ def test_float_cast_arithmetic_rounding(native_artifacts, output_dtype): dtype="float32", ) values = np.resize(samples, count) - computation = values.astype(output_dtype) - constant = np.array(0.1, dtype=output_dtype) + computation = values.astype("float64") + constant = np.float64(0.1) with np.errstate(all="ignore"): - expected = (computation + constant) - computation + expected = ((computation + constant) - computation).astype(output_dtype) for _ in range(2): actual = kernel.evaluate({"x": values}) np.testing.assert_array_equal(actual, expected) @@ -584,11 +595,9 @@ def test_float_arithmetic_constant_rounding(native_artifacts, output_dtype, oper kernel = blosc2.PortableKernel.from_json(artifact, jit=jit) assert_execution_mode(kernel, jit) values = np.resize(np.array([0.1, -1, 0, 1, 16777216], dtype="float32"), count) - # These expressions contain a floating literal: native compilation types - # that constant from the requested floating output context. Each operation - # then rounds its operands/result in that context, not C's double literal type. - arithmetic_values = values.astype(output_dtype) - constant = np.array(0.1, dtype=output_dtype) + # Operand-driven float32 computation precedes the output conversion. + arithmetic_values = values + constant = np.array(0.1, dtype="float32") if operation == "add": expected = (arithmetic_values + constant) - arithmetic_values elif operation == "subtract": @@ -596,7 +605,7 @@ def test_float_arithmetic_constant_rounding(native_artifacts, output_dtype, oper else: expected = (arithmetic_values * constant) - arithmetic_values for _ in range(2): - np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected) + np.testing.assert_array_equal(kernel.evaluate({"x": values}), expected.astype(output_dtype)) @pytest.mark.parametrize(("compiler", "jit"), [("tcc", False), ("tcc", True), ("cc", True)]) diff --git a/tests/test_portable_descriptor.py b/tests/test_portable_descriptor.py new file mode 100644 index 000000000..58a5d7caf --- /dev/null +++ b/tests/test_portable_descriptor.py @@ -0,0 +1,141 @@ +"""Opt-in installed-native descriptor fixtures; never search a sibling checkout.""" + +import json + +import numpy as np +import pytest + +from blosc2 import blosc2_ext + +pytestmark = pytest.mark.skipif( + not getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(), + reason="Installed native runtime has no draft 1.0 descriptor ABI", +) + + +def manifest(source, inputs, output, *, requires, ndim=0): + return json.dumps( + { + "schema_version": "1.0", + "language": {"name": "miniexpr", "version": "1.0"}, + "requires": requires, + "source": source, + "entry_point": "k", + "inputs": inputs, + "constants": [], + "output": output, + "semantics": {"fp": "strict"}, + "context": {"ndim": ndim}, + "metadata": {"status": "implementation-draft"}, + } + ).encode() + + +def test_descriptor_scalar_mask_empty(): + artifact = blosc2_ext.PortableArtifactHandle( + manifest( + "def k(x):\n return sum(x)\n", + [{"name": "x", "dtype": "int64"}], + {"dtype": "int64", "contract": "block_scalar"}, + requires=["numeric", "block-reductions"], + ), + 0, + ) + assert artifact.descriptor_info()["cardinality"] == "block_scalar" + result = artifact.evaluate_block( + {"x": np.array([np.iinfo(np.int64).max, 2, 3])}, + (3,), + valid_mask=np.array([False, True, True]), + ) + assert result.shape == () + assert result == 5 + assert artifact.evaluate_block({"x": np.array([], dtype="int64")}, (0,)) == 0 + assert ( + artifact.evaluate_block( + {"x": np.array([np.iinfo(np.int64).max, 2, 3])}, + (3,), + valid_mask=np.zeros(3, dtype=bool), + ) + == 0 + ) + record = json.loads( + manifest( + "def k(x):\n s = sum(x)\n return s\n", + [{"name": "x", "dtype": "int64"}], + {"dtype": "int64", "contract": "block_scalar"}, + requires=["numeric", "block-reductions"], + ) + ) + local = blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + assert ( + local.evaluate_block( + {"x": np.array([99, 2, 3], dtype="int64")}, (3,), valid_mask=np.array([False, True, True]) + ) + == 5 + ) + record["output"]["contract"] = "elementwise" + with pytest.raises(ValueError, match="cardinality"): + blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + record["output"]["contract"] = "block_scalar" + record["requires"].append("control-flow") + for body in ( + " if x < 0:\n return s + 1\n return s\n", + " for i in range(x):\n return s\n return s + 1\n", + " for i in range(3):\n if x < 0:\n break\n return s + 1\n return s\n", + " for i in range(3):\n if x < 0:\n continue\n return s + 1\n return s\n", + " while all(x != 0):\n if x < 0:\n break\n return s + 1\n return s\n", + " if s < 0:\n return s\n elif x < 0:\n return s + 1\n return s\n", + ): + record["source"] = "def k(x):\n s = sum(x)\n" + body + with pytest.raises(ValueError, match="ambiguous block-scalar"): + blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + for body in ( + " if all(x > 0):\n return s + 1\n return s\n", + " for i in range(3):\n if s > 0:\n return s + i\n return s\n", + " for i in range(3):\n if x < 0:\n break\n return s\n", + " while all(x > 0):\n return s + 1\n return s\n", + ): + record["source"] = "def k(x):\n s = sum(x)\n" + body + coherent = blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + assert coherent.evaluate_block({"x": np.array([-1, 2], dtype="int64")}, (2,)) == 1 + + +def test_descriptor_fixed_unicode(): + artifact = blosc2_ext.PortableArtifactHandle( + manifest( + "def k(x):\n return upper(x)\n", + [{"name": "x", "dtype": "unicode32", "itemsize": 16}], + {"dtype": "unicode32", "itemsize": 16, "contract": "elementwise"}, + requires=["numeric", "fixed-strings"], + ), + 0, + ) + result = artifact.evaluate_block({"x": np.array(["ß a"], dtype="U4")}, (1,)) + np.testing.assert_array_equal(result, np.array(["SS A"], dtype="U4")) + record = json.loads( + manifest( + "def k(x):\n return upper(x)\n", + [{"name": "x", "dtype": "unicode32", "itemsize": 16}], + {"dtype": "unicode32", "itemsize": 12, "contract": "elementwise"}, + requires=["numeric", "fixed-strings"], + ) + ) + with pytest.raises(ValueError, match="width"): + blosc2_ext.PortableArtifactHandle(json.dumps(record).encode(), 0) + + +def test_descriptor_logical_coordinates(): + artifact = blosc2_ext.PortableArtifactHandle( + manifest( + "def k():\n return _flat_idx\n", + [], + {"dtype": "int64", "contract": "elementwise"}, + requires=["numeric", "nd-context"], + ndim=2, + ), + 0, + ) + result = artifact.evaluate_block({}, (2, 2), logical_shape=(4, 5), block_origin=(2, 3)) + np.testing.assert_array_equal(result, [[13, 14], [18, 19]]) + with pytest.raises(ValueError, match="Explicit logical"): + artifact.evaluate_block({}, (2, 2)) diff --git a/tests/test_portable_descriptor_host.py b/tests/test_portable_descriptor_host.py new file mode 100644 index 000000000..a580ca18b --- /dev/null +++ b/tests/test_portable_descriptor_host.py @@ -0,0 +1,58 @@ +"""Host routing tests run against the installed pin without a native 1.0 engine.""" + +import numpy as np +import pytest + +from blosc2 import PortableKernel, blosc2_ext + + +def test_native_descriptor_authority_and_block_routing(monkeypatch): + calls = [] + marker = object() + + class Handle: + def __init__(self, record, jit): + assert record == b"opaque record validated only by native code" + + def info(self): + return { + "source": "native source", + "entry_point": "k", + "inputs": {}, + "output_dtype": None, + "jit": False, + } + + def descriptor_info(self): + return { + "schema_version": "1.0", + "inputs": {"x": np.dtype("int64")}, + "output_dtype": np.dtype("int64"), + "cardinality": "block_scalar", + "ndim": 2, + } + + def evaluate_block(self, arrays, shape, **context): + calls.append((arrays, shape, context)) + return marker + + monkeypatch.setattr(blosc2_ext, "PortableArtifactHandle", Handle) + monkeypatch.setattr(blosc2_ext, "portable_descriptor_available", lambda: True, raising=False) + kernel = PortableKernel.from_json(b"opaque record validated only by native code") + assert kernel.result_cardinality == "block_scalar" + assert kernel.context_ndim == 2 + with pytest.raises(ValueError, match="explicit logical_shape"): + kernel.evaluate({"x": np.arange(4, dtype="int64").reshape(2, 2)}) + values = np.arange(8, dtype=">i8").reshape(2, 4)[:, ::2] + mask = np.array([[True, False], [True, True]]) + assert ( + kernel.evaluate_block({"x": values}, logical_shape=(4, 5), block_origin=(2, 3), valid_mask=mask) + is marker + ) + arrays, shape, context = calls[0] + assert shape == (2, 2) + assert arrays["x"].flags.c_contiguous + assert arrays["x"].dtype.isnative + assert context["logical_shape"] == (4, 5) + assert context["block_origin"] == (2, 3) + assert context["valid_mask"] is mask diff --git a/tests/test_portable_lazy.py b/tests/test_portable_lazy.py new file mode 100644 index 000000000..6ccee5a50 --- /dev/null +++ b/tests/test_portable_lazy.py @@ -0,0 +1,148 @@ +import numpy as np +import pytest +from test_portable_descriptor import manifest + +import blosc2 +from blosc2 import blosc2_ext + +pytestmark = pytest.mark.skipif( + not getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(), + reason="Installed runtime has no draft descriptor ABI", +) + + +def kernel(source, *, scalar=False, nd=False, string=False): + dtype = "unicode32" if string else "int64" + inputs = [{"name": "x", "dtype": dtype}] + output = {"dtype": dtype, "contract": "block_scalar" if scalar else "elementwise"} + requires = ["numeric"] + if string: + inputs[0]["itemsize"] = output["itemsize"] = 16 + requires.append("fixed-strings") + if scalar: + requires.append("block-reductions") + if nd: + requires.append("nd-context") + return blosc2.PortableKernel.from_json( + manifest(source, inputs, output, requires=requires, ndim=2 if nd else 0), jit=False + ) + + +@pytest.mark.parametrize("scalar", [False, True]) +def test_logical_groups_partial_and_rechunk(tmp_path, scalar): + values = np.arange(35, dtype="int64").reshape(5, 7) + stored = blosc2.asarray(values, chunks=(3, 4), blocks=(1, 2), urlpath=tmp_path / "input.b2nd") + k = kernel( + "def k(x):\n return sum(x)\n" if scalar else "def k(x):\n return x + _flat_idx\n", + scalar=scalar, + nd=not scalar, + ) + lazy = k.lazy({"x": stored}, partitions=(2, 3)) + expected = ( + np.array([[values[i : i + 2, j : j + 3].sum() for j in range(0, 7, 3)] for i in range(0, 5, 2)]) + if scalar + else values * 2 + ) + assert lazy.partitions == (2, 3) + np.testing.assert_array_equal(lazy[:], expected) + np.testing.assert_array_equal(lazy[1:, ::2], expected[1:, ::2]) + np.testing.assert_array_equal( + lazy.compute(item=(slice(1, None), slice(None, None, 2)))[:], expected[1:, ::2] + ) + np.testing.assert_array_equal(lazy.rechunk(chunks=(1, 2), blocks=(1, 1))[:], expected) + path = tmp_path / "lazy.b2nd" + lazy.save(path) + reopened = blosc2.open(path) + assert reopened.partitions == (2, 3) + np.testing.assert_array_equal(reopened[:], expected) + mixed = lazy + 3 + mixed.save(tmp_path / "mixed.b2nd") + np.testing.assert_array_equal(blosc2.open(tmp_path / "mixed.b2nd")[:], expected + 3) + + +def test_string_roundtrip_and_native_only_load(tmp_path, monkeypatch): + values = np.array(["ß a", "ab", "z"], dtype="U4") + stored = blosc2.asarray(values, urlpath=tmp_path / "text.b2nd") + lazy = kernel("def k(x):\n return upper(x)\n", string=True).lazy({"x": stored}, partitions=(2,)) + lazy.save(tmp_path / "textlazy.b2nd") + import importlib + + dsl_module = importlib.import_module("blosc2.dsl_kernel") + monkeypatch.setattr( + dsl_module, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + import blosc2.b2objects as b2objects + + monkeypatch.setattr( + b2objects, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + np.testing.assert_array_equal(blosc2.open(tmp_path / "textlazy.b2nd")[:], ["SS A", "AB", "Z"]) + + +def test_preflight_preserves_destination(tmp_path): + lazy = kernel("def k(x):\n return x + 1\n").lazy({"x": np.arange(3, dtype="int64")}) + path = tmp_path / "existing" + path.write_bytes(b"sentinel") + with pytest.raises((TypeError, ValueError)): + lazy.save(path) + assert path.read_bytes() == b"sentinel" + + +@pytest.mark.parametrize("route", ["frame", "msgpack", "embed", "dict", "tree"]) +def test_container_routes(tmp_path, route): + from blosc2.msgpack_utils import msgpack_packb, msgpack_unpackb + + source = blosc2.asarray(np.arange(9, dtype="int64"), urlpath=tmp_path / "source.b2nd") + lazy = kernel("def k(x):\n return sum(x)\n", scalar=True).lazy({"x": source}, partitions=(4,)) + expected = np.array([6, 22, 8]) + if route == "frame": + reopened = blosc2.from_cframe(lazy.to_cframe()) + elif route == "msgpack": + reopened = msgpack_unpackb(msgpack_packb(lazy), deserialize="safe") + else: + cls = {"embed": blosc2.EmbedStore, "dict": blosc2.DictStore, "tree": blosc2.TreeStore}[route] + with cls(str(tmp_path / "store.b2z"), mode="w") as store: + store["/lazy"] = lazy + reopened = store["/lazy"] + np.testing.assert_array_equal(reopened[:], expected) + with cls(str(tmp_path / "store.b2z"), mode="r") as store: + reopened = store["/lazy"] + np.testing.assert_array_equal(reopened[:], expected) + return + np.testing.assert_array_equal(reopened[:], expected) + + +def test_partial_reads_only_original_groups(): + class Operand: + dtype = np.dtype("int64") + shape = (12,) + + def __init__(self): + self.reads = [] + + def __getitem__(self, item): + self.reads.append(item) + return np.arange(12, dtype="int64")[item] + + operand = Operand() + lazy = kernel("def k(x):\n return sum(x)\n", scalar=True).lazy({"x": operand}, partitions=(4,)) + np.testing.assert_array_equal(lazy[1:2], [22]) + assert operand.reads == [(slice(4, 8),)] + assert lazy[1] == 22 + np.testing.assert_array_equal(lazy[::-1], [38, 22, 6]) + author = blosc2.DSLKernel.from_source("def k(x):\n return x + sum(x)\n") + elementwise = blosc2.PortableKernel.from_json(author.export({"x": "int64"}, "int64", version="1.0")) + grouped = elementwise.lazy({"x": operand}, partitions=(4,)) + operand.reads.clear() + np.testing.assert_array_equal(grouped[7:3:-2], [29, 27]) + assert operand.reads == [(slice(4, 8),)] + np.testing.assert_array_equal(grouped[4:4], np.array([], dtype="int64")) + assert operand.reads == [(slice(4, 8),)] + + +def test_broadcast_coordinates_keep_output_domain(): + values = np.arange(7, dtype="int64").reshape(1, 7) + k = kernel("def k(x):\n return x + _flat_idx\n", nd=True) + lazy = k.lazy({"x": values}, shape=(5, 7), partitions=(2, 3)) + expected = values + np.arange(35).reshape(5, 7) + np.testing.assert_array_equal(lazy[3:, 4:], expected[3:, 4:]) diff --git a/tests/test_portable_policy.py b/tests/test_portable_policy.py new file mode 100644 index 000000000..1250e645e --- /dev/null +++ b/tests/test_portable_policy.py @@ -0,0 +1,105 @@ +"""Metadata policy/preflight checks independent of the installed native version.""" + +from pathlib import Path + +import numpy as np +import pytest + +import blosc2 +from blosc2.b2objects import decode_b2object_payload + + +@pytest.mark.parametrize("policy", ["safe", "full"]) +def test_nested_legacy_effective_policy(tmp_path, policy): + fixture = Path(__file__).parent / "data" / "legacy_lazyudf_v1" / "expr.b2nd" + carrier = blosc2.empty((5,), dtype="float64", meta={"b2o": {"kind": "lazyexpr", "version": 1}}) + carrier.schunk.vlmeta["b2o"] = { + "kind": "lazyexpr", + "version": 1, + "expression": "x + 1", + "operands": {"x": {"kind": "urlpath", "version": 1, "urlpath": str(fixture)}}, + } + if policy == "safe": + with pytest.raises(blosc2.UnsafeDeserializationError): + blosc2.from_cframe(carrier.to_cframe(), deserialize=policy) + else: + opened = blosc2.from_cframe(carrier.to_cframe(), deserialize=policy) + nested = next(iter(opened.operands.values())) + assert isinstance(nested, blosc2.LazyUDF) + np.testing.assert_allclose(nested.compute()[:], (np.arange(5) * 3) ** 2) + + +def test_structured_legacy_rejects_before_python(monkeypatch): + from msgpack import ExtType, packb + + import blosc2.b2objects as b2objects + from blosc2.msgpack_utils import msgpack_unpackb + + record = { + "kind": "lazyudf", + "version": 1, + "function_kind": "dsl", + "dsl_version": 1, + "udf_source": "def k():\n return 1\n", + "name": "k", + "dtype": "int64", + "shape": [2], + "operands": {}, + "kwargs": {}, + } + legacy = ExtType(43, packb(record)) + nested = ExtType(46, packb({"values": [legacy], "shape": [1]})) + for payload in (packb(legacy), packb(nested)): + trusted = msgpack_unpackb(payload, deserialize="full") + assert isinstance(trusted if isinstance(trusted, blosc2.LazyUDF) else trusted[0], blosc2.LazyUDF) + legacy_recipe = trusted if isinstance(trusted, blosc2.LazyUDF) else trusted[0] + with pytest.raises(TypeError, match="legacy"): + legacy_recipe.to_cframe() + + monkeypatch.setattr(b2objects, "kernel_from_source", lambda *a, **k: pytest.fail("Policy bypass")) + with pytest.raises(blosc2.UnsafeDeserializationError): + decode_b2object_payload({"kind": "lazyudf", "version": 1}, deserialize="safe") + for payload in (packb(legacy), packb(nested)): + with pytest.raises(blosc2.UnsafeDeserializationError): + msgpack_unpackb(payload) + with pytest.raises(blosc2.UnsafeDeserializationError): + msgpack_unpackb(payload, deserialize="safe") + + +@pytest.mark.parametrize("route", ["embed", "dict", "tree"]) +def test_rejected_kernel_does_not_replace_store_value(tmp_path, route): + cls = {"embed": blosc2.EmbedStore, "dict": blosc2.DictStore, "tree": blosc2.TreeStore}[route] + udf = blosc2.lazyudf(lambda inputs, output, offset: None, (np.arange(4),), dtype="int64") + with cls(str(tmp_path / "store.b2z"), mode="w") as store: + store["/value"] = np.arange(4) + with pytest.raises(TypeError, match="PortableKernel"): + store["/value"] = udf + np.testing.assert_array_equal(store["/value"][:], np.arange(4)) + operand = blosc2.asarray(np.arange(4), urlpath=tmp_path / "input.b2nd", mode="w") + for body in ("print(x)\n return x", "return callback(x)"): + kernel = blosc2.DSLKernel.from_source(f"def invalid(x):\n {body}\n") + invalid = blosc2.lazyudf(kernel, (operand,), dtype="int64") + with pytest.raises(blosc2.PortableArtifactError): + store["/value"] = invalid + np.testing.assert_array_equal(store["/value"][:], np.arange(4)) + valid = blosc2.DSLKernel.from_source("def valid(x):\n return x + 2\n") + store["/recipe"] = blosc2.lazyudf(valid, (operand,), dtype="int64") + np.testing.assert_array_equal(store["/recipe"][:], np.arange(4) + 2) + + +def test_ctable_legacy_metadata_policy_before_reconstruction(monkeypatch): + from types import SimpleNamespace + + table = object.__new__(blosc2.CTable) + table._storage = SimpleNamespace(_deserialize_mode="safe") + schema = { + "computed_columns": [ + {"name": "bad", "col_deps": [], "dtype": "int64", "kind": "dsl", "dsl_source": "invalid"} + ] + } + with pytest.raises(blosc2.UnsafeDeserializationError): + table._load_computed_cols_from_schema(schema) + table._computed_cols = {"x": {"kind": "dsl"}} + table._materialized_cols = {} + with pytest.raises(TypeError, match="row-domain"): + table._preflight_portable_persistence() diff --git a/tests/test_portable_table_export.py b/tests/test_portable_table_export.py new file mode 100644 index 000000000..b0ff79033 --- /dev/null +++ b/tests/test_portable_table_export.py @@ -0,0 +1,221 @@ +import importlib +from dataclasses import dataclass + +import numpy as np +import pytest + +import blosc2 +from blosc2 import blosc2_ext + +pytestmark = pytest.mark.skipif( + not getattr(blosc2_ext, "portable_descriptor_available", lambda: False)(), + reason="Installed runtime has no draft descriptor ABI", +) + + +def test_authoring_typed_snapshots(monkeypatch, tmp_path): + validate = blosc2.validate_portable_dsl + with monkeypatch.context() as guard: + guard.setattr( + blosc2_ext, "PortableArtifactHandle", lambda *a: pytest.fail("Artifact loader invoked") + ) + cache = tmp_path / "validation-jit-cache" + guard.setenv("CC", "/no/compiler") + guard.setenv("ME_DSL_JIT_COMPILER", "cc") + guard.setenv("ME_DSL_JIT", "1") + guard.setenv("ME_DSL_JIT_CACHE_DIR", str(cache)) + assert validate( + "# me:compiler=cc\ndef k(x):\n while 1:\n pass\n return x\n", + {"x": "float32"}, + "float32", + language_version="1.0", + )["valid"] + assert validate( + "# me:compiler=cc\ndef k(x, y):\n return sin(x) + y\n", + {"y": "int16", "x": "float32"}, + "float32", + language_version="1.0", + )["valid"] + assert validate("def k(x):\n return upper(x)\n", {"x": "U4"}, "U4", language_version="1.0")[ + "valid" + ] + assert validate("def k(x):\n return x\n", {"x": ">S4"}, "S4", language_version="1.0")["valid"] + assert not validate("def k(x):\n return upper(x)\n", {"x": "U4"}, "U3", language_version="1.0")[ + "valid" + ] + scalar = "def k(x):\n return sum(x)\n" + assert validate(scalar, {"x": "int16"}, "int64", language_version="1.0", cardinality="block_scalar")[ + "valid" + ] + assert ( + validate(scalar, {"x": "int16"}, "int64", language_version="1.0", cardinality="elementwise")[ + "status" + ] + == "invalid_signature" + ) + nd = "def k(x):\n return x + _i1\n" + missing = validate(nd, {"x": "int64"}, "int64", language_version="1.0") + assert not missing["valid"] + assert "rank" in missing["error"] + assert validate(nd, {"x": "int64"}, "int64", language_version="1.0", ndim=2)["valid"] + pair = {"x": "int64", "y": "uint64"} + assert validate("def k(x, y):\n return x < y\n", pair, "bool", language_version="1.0")["valid"] + assert not validate("def k(x, y):\n return x + y\n", pair, "float64", language_version="1.0")[ + "valid" + ] + for body in ("print(x)\n return x", "return callback(x)", "return x & 1"): + rejected = validate( + f"def k(x):\n {body}\n", {"x": "float32"}, "float32", language_version="1.0" + ) + assert not rejected["valid"] + assert rejected["line"] == 2 + assert rejected["column"] > 0 + invalid = validate("def k(x):\n return (\n", {"x": "float32"}, "float32", language_version="1.0") + assert invalid["status"] == "invalid_source" + assert invalid["line"] > 0 + assert invalid["column"] > 0 + assert not cache.exists() + + offset = np.uint64(7) + + @blosc2.dsl_kernel + def author(x): + return x + offset + + record = author.export({"x": "uint64"}, "uint64", version="1.0") + offset = np.uint64(99) + kernel = blosc2.PortableKernel.from_json(record) + np.testing.assert_array_equal(kernel.evaluate({"x": np.arange(3, dtype="uint64")}), [7, 8, 9]) + text = "ß" + + @blosc2.dsl_kernel + def string_author(x): + return x + text + + record = string_author.export({"x": "U2"}, "U3", version="1.0") + text = "changed" + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(record).evaluate({"x": np.array(["a"], dtype="U2")}), ["aß"] + ) + static = blosc2.DSLKernel.from_source("def rows(x):\n return sum(x) + _n0\n") + record = static.export({"x": "int64"}, "int64", version="1.0", cardinality="block_scalar", ndim=1) + assert blosc2.PortableKernel.from_json(record).result_cardinality == "block_scalar" + + minimum = np.int64(np.iinfo(np.int64).min) + + @blosc2.dsl_kernel + def minimum_capture(x): + return x + minimum + + record = minimum_capture.export( + {"x": "float64"}, "float64", version="1.0", capture_dtypes={"minimum": "float64"} + ) + minimum = np.int64(0) + np.testing.assert_array_equal( + blosc2.PortableKernel.from_json(record).evaluate({"x": np.array([0.0])}), [-float(2**63)] + ) + loop = blosc2.DSLKernel.from_source("def k(x):\n for i in range(x):\n y = i\n return i\n") + native_loop = blosc2.PortableKernel.from_json(loop.export({"x": "int64"}, "int64", version="1.0")) + np.testing.assert_array_equal(native_loop.evaluate({"x": np.array([2, 4], dtype="int64")}), [1, 3]) + + @blosc2.dsl_kernel + def static_row(row): + return np.sqrt(row["x"]) + row["y"] + + bound = dict(zip(static_row.input_names, ("float64", "float64"), strict=True)) + native = blosc2.PortableKernel.from_json(static_row.export(bound, "float64", version="1.0")) + arrays = dict(zip(static_row.input_names, (np.array([4.0]), np.array([3.0])), strict=True)) + np.testing.assert_array_equal(native.evaluate(arrays), [5.0]) + with pytest.raises(blosc2.PortableArtifactError): + author.export({"x": "uint64"}, "uint64", version="1.0", constants={"unknown": 1}) + + +def test_table_native_row_contract_roundtrip(tmp_path, monkeypatch): + @dataclass + class Row: + x: int = 0 + + source = blosc2.DSLKernel.from_source("def rows(x):\n return sum(x) + _flat_idx + _n0\n") + kernel = blosc2.PortableKernel.from_json(source.export({"x": "int64"}, "int64", version="1.0", ndim=1)) + table = blosc2.CTable(Row, new_data={"x": [2, 4]}, create_summary_index=False) + table.add_portable_computed_column("virtual", kernel, inputs={"x": "x"}, row_domain="independent") + table.add_portable_generated_column( + "stored", source, dtype="int64", inputs={"x": "x"}, row_domain="independent" + ) + np.testing.assert_array_equal(table["virtual"][:], [3, 5]) + table.append({"x": 8}) + np.testing.assert_array_equal(table["stored"][:], [3, 5, 9]) + table.refresh_generated_column("stored") + path = tmp_path / "table.b2d" + table.save(str(path)) + import blosc2.b2objects as b2objects + + monkeypatch.setattr( + b2objects, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + # The public package attribute is a decorator; import the actual module. + dsl_module = importlib.import_module("blosc2.dsl_kernel") + monkeypatch.setattr( + dsl_module, "kernel_from_source", lambda *a, **k: pytest.fail("Python reconstruction") + ) + reopened = blosc2.CTable.open(str(path), mode="a") + np.testing.assert_array_equal(reopened["virtual"][:], [3, 5, 9]) + reopened.append({"x": 10}) + np.testing.assert_array_equal(reopened["stored"][:], [3, 5, 9, 11]) + framed = blosc2.ctable_from_cframe(reopened.to_cframe()) + np.testing.assert_array_equal(framed["stored"][:], [3, 5, 9, 11]) + reopened.materialize_computed_column("virtual", new_name="derived") + reopened.delete(1) + copied = reopened.copy(compact=True) + np.testing.assert_array_equal(copied["virtual"][:], [3, 9, 11]) + np.testing.assert_array_equal(copied["derived"][:], [3, 9, 11]) + copied.extend({"x": [12, 14]}) + np.testing.assert_array_equal(copied["stored"][:], [3, 9, 11, 13, 15]) + constructor = blosc2.DSLKernel.from_source("def constant_row():\n return _n0\n") + constructed = blosc2.PortableKernel.from_json(constructor.export({}, "int64", version="1.0", ndim=1)) + copied.add_portable_generated_column("constant", constructed, inputs={}, row_domain="independent") + copied.append({"x": 16}) + copied.refresh_generated_column("constant") + np.testing.assert_array_equal(copied["constant"][:], np.ones(6, dtype="int64")) + sentinel = tmp_path / "preserved.b2z" + sentinel.write_bytes(b"original destination") + copied._computed_cols["virtual"]["artifact"] = "{}" + with pytest.raises(blosc2.PortableArtifactError): + copied.save(str(sentinel)) + assert sentinel.read_bytes() == b"original destination" + with pytest.raises(ValueError, match="independent"): + table.add_portable_computed_column("invalid", kernel, inputs={"x": "x"}, row_domain="dynamic") + + @dataclass + class TextRow: + label: str = blosc2.field(blosc2.string(max_length=4), default="") + + text_table = blosc2.CTable(TextRow, new_data={"label": ["ß a", "ab"]}, create_summary_index=False) + upper = blosc2.DSLKernel.from_source("def upper_row(x):\n return upper(x)\n") + text_table.add_portable_generated_column( + "upper", upper, dtype="U4", inputs={"x": "label"}, row_domain="independent" + ) + text_table.append({"label": "z"}) + np.testing.assert_array_equal(text_table["upper"][:], ["SS A", "AB", "Z"]) + + @dataclass + class VectorRow: + vector: object = blosc2.field(blosc2.ndarray((3,), dtype=blosc2.int64())) + bias: int = 0 + + vector_table = blosc2.CTable( + VectorRow, new_data={"vector": [[1, 2, 3], [4, 5, 6]], "bias": [1, 2]}, create_summary_index=False + ) + total = blosc2.DSLKernel.from_source("def total_row(x, bias):\n return sum(x + bias) + _n0\n") + vector_table.add_portable_generated_column( + "total", + total, + dtype="int64", + cardinality="block_scalar", + inputs={"x": "vector", "bias": "bias"}, + row_domain="independent", + ) + vector_table.append({"vector": [7, 8, 9], "bias": 3}) + vector_table.refresh_generated_column("total") + restored = blosc2.ctable_from_cframe(vector_table.to_cframe()) + np.testing.assert_array_equal(restored["total"][:], [12, 24, 36]) From 1f68b4392ed1160afbbc47551c6d6e7fdb285bf2 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 7 Oct 2026 14:02:38 +0200 Subject: [PATCH 25/30] Prepare Menudet 1.0 draft release certification --- .github/workflows/build.yml | 11 + .github/workflows/cibuildwheels.yml | 12 + .github/workflows/wasm-tests.yml | 4 +- CMakeLists.txt | 2 +- RELEASE_NOTES.md | 31 + doc/conf.py | 11 +- doc/getting_started/installation.rst | 10 +- doc/reference/dsl_syntax.md | 4 + doc/reference/index.rst | 1 + doc/reference/menudet_accuracy.md | 52 + doc/reference/misc.rst | 5 +- doc/reference/portable_dsl.rst | 33 +- doc/reference/save_load.rst | 11 + examples/menudet.py | 32 + plans/portable-DSL-1.0-implemenation.md | 11 +- plans/portable-DSL-1.0-progress.md | 137 ++ pyproject.toml | 2 +- scripts/certify_menudet_math.py | 174 ++ scripts/menudet_math_vectors.json | 2622 +++++++++++++++++++++++ src/blosc2/version.py | 2 +- 20 files changed, 3141 insertions(+), 26 deletions(-) create mode 100644 doc/reference/menudet_accuracy.md create mode 100644 examples/menudet.py create mode 100644 scripts/certify_menudet_math.py create mode 100644 scripts/menudet_math_vectors.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 868b01763..43c7ea405 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,6 +108,17 @@ jobs: PYTEST_DEADMAN_SECONDS: "300" PYTEST_DEADMAN_SESSION_SECONDS: "1500" + - name: Certify bounded Menudet math against independent references + run: python scripts/certify_menudet_math.py --report menudet-math.json + + - name: Upload Menudet math certification + if: always() + uses: actions/upload-artifact@v4 + with: + name: menudet-math-${{ matrix.os }}-py${{ matrix.python-version }}-np${{ matrix.numpy-version || 'default' }} + path: menudet-math.json + if-no-files-found: ignore + # TUIs are timing-sensitive and only boot a headless terminal: one OS is # enough, and a flake here is feedback, not a release blocker. - name: Test TUI (non-blocking) diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml index ef3c26092..99648a9d4 100644 --- a/.github/workflows/cibuildwheels.yml +++ b/.github/workflows/cibuildwheels.yml @@ -11,6 +11,18 @@ on: env: CIBW_BUILD_VERBOSITY: 1 + # Every tested wheel checks portable descriptors/persistence and independent + # numeric references, not only whether the extension can be imported. + CIBW_TEST_COMMAND: >- + python -m pytest + {project}/tests/test_portable_artifact.py + {project}/tests/test_portable_descriptor.py + {project}/tests/test_portable_descriptor_host.py + {project}/tests/test_portable_lazy.py + {project}/tests/test_portable_policy.py + {project}/tests/test_portable_table_export.py + {project}/tests/test_dsl_portable.py -q -n 0 && + python {project}/scripts/certify_menudet_math.py # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long # Now that github provides native arm64 runners, we can enable tests again # CIBW_TEST_SKIP: "*linux*aarch64*" diff --git a/.github/workflows/wasm-tests.yml b/.github/workflows/wasm-tests.yml index c9ef42595..211506b06 100644 --- a/.github/workflows/wasm-tests.yml +++ b/.github/workflows/wasm-tests.yml @@ -19,7 +19,9 @@ jobs: # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF" # cibuildwheel runs the suite automatically in the node/pyodide runtime - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: >- + pytest {project}/tests && + python {project}/scripts/certify_menudet_math.py steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 diff --git a/CMakeLists.txt b/CMakeLists.txt index bc12f9b3c..e31cd81f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,7 +142,7 @@ endif() FetchContent_Declare(miniexpr GIT_REPOSITORY https://github.com/Blosc/miniexpr.git - GIT_TAG 55c882b238c4daf0523a52a44d5d84db56af24e4 + GIT_TAG 3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4 # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr ) FetchContent_MakeAvailable(miniexpr) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1d7bd308c..a91a5186f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,36 @@ # Release notes +## Changes from 4.14.2 to 4.15.0 (in development) + +### Introducing the Menudet 1.0 draft + +**Menudet — a little language for portable computation on arrays and tables.** + +- Export normalized `DSLKernel` authoring as validated, standalone JSON artifacts + with typed signatures and exact captured constants. Native miniexpr executes + imports without reconstructing Python functions or falling back to Python. +- Support checked integer arithmetic, operand-typed floating computation, control + flow, fixed-width strings, logical array coordinates and ordered block reductions. +- Keep logical reduction groups stable through persistence, partial reads and + output rechunking. True block-scalar results expose the logical block-grid shape. +- Add portable CTable computed/generated transformers with an explicit + `row_domain="independent"` contract, including fixed-shape row reductions. +- New compliant DSLKernel-backed LazyUDF saves export the draft automatically; + unsupported saves fail validation before writing the destination. +- Historical Python-specific recipes require explicit `deserialize="full"`. + MessagePack decoding is safe by default, including nested containers. + +Menudet 1.0 is a **draft**, not a frozen compatibility promise. The existing +`DSLKernel` and portable API names remain unchanged. This draft uses the typed +interpreter; optional JIT requests fall back before execution, while required JIT +rejects unsupported acceleration. Complex computation, callbacks, arbitrary +memory access and non-strict floating-point modes are outside the portable contract. + +See the [Menudet reference](https://www.blosc.org/python-blosc2/reference/portable_dsl.html) +for array/table examples, persistence policy and current limitations. Release +certification is tracked separately; this development entry is not an announcement +that 4.15.0 has been published. + ## Changes from 4.14.1 to 4.14.2 XXX version-specific blurb XXX diff --git a/doc/conf.py b/doc/conf.py index 915bb0c20..b709da445 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -111,7 +111,14 @@ def genbody(f, func_list, lib="blosc2"): ], } -exclude_patterns = ["_build", ".DS_Store", "**.ipynb_checkpoints", "tutorials/images/**"] +exclude_patterns = [ + "_build", + ".DS_Store", + "**.ipynb_checkpoints", + "tutorials/images/**", + "html/**", + "**/autofiles-stash/**", +] html_show_sourcelink = False @@ -306,7 +313,7 @@ def process_sig(app, what, name, obj, options, signature, return_annotation): # ``-W`` (as CI does) to turn that warning into a failure. _AUTODOC_DIRECTIVE = re.compile( - r"^\s*\.\.\s+(?:autoclass|autofunction|autodata|autoexception|autodecorator)::" + r"^\s*\.\.\s+(?:autoclass|autofunction|autodata|autoexception|autodecorator|py:class)::" r"\s*([\w.]+)" ) _CURRENTMODULE = re.compile(r"^\s*\.\.\s+(?:currentmodule|module)::\s*([\w.]+)") diff --git a/doc/getting_started/installation.rst b/doc/getting_started/installation.rst index fd0e689de..7ba918308 100644 --- a/doc/getting_started/installation.rst +++ b/doc/getting_started/installation.rst @@ -33,13 +33,13 @@ grouped into *extras* that you opt into with the ``blosc2[extra]`` syntax: - The :doc:`b2view <../guides/b2view>` terminal browser (``textual``, ``textual-plotext``), including its in-terminal braille plot (the ``p`` key). Required by the ``b2view`` command. - * - ``hires`` + * - ``hires`` - The high-resolution image view in b2view (the ``h`` key), which renders a real ``matplotlib`` image in the terminal - (``textual-image``, ``matplotlib``). Includes ``tui``. - * - ``images`` - - JPEG/PNG ordinary-file previews in b2view (``Pillow``, ``textual-image``). - Includes ``tui`` without requiring matplotlib. ``hires`` includes this extra. + (``textual-image``, ``matplotlib``). Includes ``tui``. + * - ``images`` + - JPEG/PNG ordinary-file previews in b2view (``Pillow``, ``textual-image``). + Includes ``tui`` without requiring matplotlib. ``hires`` includes this extra. * - ``parquet`` - The ``parquet-to-blosc2`` converter (``pyarrow``); see :doc:`../guides/parquet_to_blosc2`. diff --git a/doc/reference/dsl_syntax.md b/doc/reference/dsl_syntax.md index 308ae7ae9..71e22deb5 100644 --- a/doc/reference/dsl_syntax.md +++ b/doc/reference/dsl_syntax.md @@ -3,6 +3,10 @@ This is the practical reference for the DSL used by `@blosc2.dsl_kernel` functions (and checked by `blosc2.validate_dsl()`). It focuses on what works today and the most common gotchas. +The portable subset is named **Menudet**, a little language for portable +computation on arrays and tables. See the [Menudet reference](portable_dsl.rst) +for the 1.0 draft contract. This page also documents ordinary full-DSL features; +acceptance here does not imply portability. For usage walkthroughs and end-to-end examples, see the [LazyArray UDF DSL kernels tutorial](../tutorials/03.lazyarray-udf-kernels.ipynb). diff --git a/doc/reference/index.rst b/doc/reference/index.rst index f7d3e74d4..6fc2636c6 100644 --- a/doc/reference/index.rst +++ b/doc/reference/index.rst @@ -13,4 +13,5 @@ API Reference utilities dsl_syntax portable_dsl + menudet_accuracy jit diff --git a/doc/reference/menudet_accuracy.md b/doc/reference/menudet_accuracy.md new file mode 100644 index 000000000..70bb2f189 --- /dev/null +++ b/doc/reference/menudet_accuracy.md @@ -0,0 +1,52 @@ +# Menudet draft: numeric certification + +Menudet requires checked integer results and the stated floating-point type, +domain, NaN, infinity, signed-zero and rounding rules. It does **not** promise +bit-identical transcendental results across host math libraries or a universal +ULP error bound. Native exact-anchor and exceptional-value tests remain separate +from the bounded independent math check described here. + +## Reproducible independent references + +`scripts/menudet_math_vectors.json` contains 284 samples for 40 canonical +operations, covering both float32 and float64. Inputs and expected results are +stored as exact hexadecimal encodings. References were generated with mpmath +1.3.0 at both 100 and 200 decimal digits; regeneration rejects disagreement +between the two rounded results. The checker does not need mpmath and does not +use NumPy or the executing host's libm as its mathematical reference. + +The finite domains include inverse-function endpoints, small arguments that +exercise cancellation, mixed quadrants, gamma functions, and explicit fused +multiply-add cancellation. Domain errors, NaNs/infinities, subnormal behavior, +signed zeros, rounding functions, nextafter and aliases additionally have native +conformance fixtures; they are not certified merely by these finite samples. + +| Operations | Maximum error for the checked samples | +| --- | --- | +| cospi, sinpi, exp10, lgamma, tgamma, logaddexp | 8 ULP | +| Other sampled transcendental functions, sqrt, hypot, floating pow | 4 ULP | +| fdim, fmin, fmax, copysign, fmod, remainder, ldexp, fma | Exact rounded value | + +These are **function-specific acceptance budgets for this finite corpus**, not +new normative bounds over all possible inputs. ULP is measured using the smaller +adjacent gap at the expected value, including at powers of two. Adding a new +sample or changing a budget requires review; a failed platform must not silently +skip a function or inflate its tolerance. + +## Running certification + +Against an installed package: + +```console +python scripts/certify_menudet_math.py --report menudet-math.json +``` + +Reports identify the package path/version, platform, reference generator, sample +count, per-function/per-precision maxima and individual failures. The process +exits nonzero on failure. Use `--generate` only to regenerate the checked-in +reference corpus in a development environment with mpmath installed. + +Native Python CI uploads the reports. The same corpus runs in Pyodide and in the +installed-wheel test command, without adding a runtime reference-library +dependency. Passing a local wheel certifies only that tested wheel and host; +release readiness additionally requires the supported platform matrix to pass. diff --git a/doc/reference/misc.rst b/doc/reference/misc.rst index e9ce82962..e91d413bf 100644 --- a/doc/reference/misc.rst +++ b/doc/reference/misc.rst @@ -7,7 +7,10 @@ This page documents the miscellaneous members of the ``blosc2`` module that do n .. autodata:: cpu_info -.. autoclass:: finfo +.. py:class:: finfo(dtype) + + Return floating-point limits for ``dtype``. This is an alias for + :class:`numpy.finfo`; see NumPy's reference for its attributes and examples. .. autoclass:: iinfo diff --git a/doc/reference/portable_dsl.rst b/doc/reference/portable_dsl.rst index b760fadaa..a2c496286 100644 --- a/doc/reference/portable_dsl.rst +++ b/doc/reference/portable_dsl.rst @@ -1,13 +1,22 @@ -Portable DSL artifacts -====================== +Menudet: portable computation +============================= + +**Menudet — a little language for portable computation on arrays and tables.** + +Menudet is the language; miniexpr is its native implementation, and Python-Blosc2 +provides authoring, array scheduling and table integration. The existing +``DSLKernel``, ``PortableKernel`` and ``validate_portable_dsl`` API names remain +unchanged. Python-like authoring syntax normalizes to a language-independent +artifact; a Menudet artifact is not an executable Python recipe. .. warning:: - Portable DSL 1.0 is experimental and its language and artifact specifications - are drafts. Local development integration is tested; publication, independent - numeric accuracy and release-platform certification remain pending. + Menudet 1.0 is experimental. Its language and artifact specifications are + drafts, not a frozen compatibility promise. Native cross-platform CI passes; + independent numeric accuracy and Python release-wheel certification are + tracked separately. Beta feedback will inform the final compatibility contract. -Portable DSL artifacts use the versioned native 1.0 profile. +Menudet artifacts use the versioned native 1.0 profile. Artifacts are standalone UTF-8 JSON, executable through native miniexpr in C or Python, without reconstructing or executing Python functions on import. @@ -29,6 +38,10 @@ parameters may have different dtypes. Captured Python/NumPy scalars are snapshot typed constants, never live callbacks or arrays. Integers use range-checked decimal strings; floats use exact IEEE-754 hexadecimal bits. +For a runnable array/table example using the same kernel, see +``examples/menudet.py`` in the source distribution. Numeric certification and its +finite-domain limits are described in :doc:`menudet_accuracy`. + Release boundary ---------------- @@ -193,10 +206,10 @@ Semantic evaluation errors never retry the kernel. Artifacts enforce strict FP independently of host defaults. Host while-loop limits remain execution policy rather than artifact constants. -The authoritative language and artifact specifications are maintained in the -miniexpr development sources under ``doc/dsl-spec/1.0.md`` and -``doc/dsl-spec/artifact-1.0.md``. The published dependency pin has not yet changed; -the local development override is not stock-wheel or platform certification. +The authoritative language and artifact specifications are maintained in +miniexpr under ``doc/dsl-spec/1.0.md`` and ``doc/dsl-spec/artifact-1.0.md``. +Python-Blosc2 pins the published native revision that passed the native platform +matrix. Native CI is distinct from Python stock-wheel certification. API --- diff --git a/doc/reference/save_load.rst b/doc/reference/save_load.rst index 493447210..16304a1c1 100644 --- a/doc/reference/save_load.rst +++ b/doc/reference/save_load.rst @@ -21,3 +21,14 @@ Save and load .. autofunction:: save_tensor .. autofunction:: load_tensor .. autofunction:: from_cframe + +Deserialization policy +~~~~~~~~~~~~~~~~~~~~~~ + +Validated Menudet artifacts load natively without reconstructing Python code. +Historical Python-specific recipes require explicit ``deserialize="full"``; +the selected policy propagates through nested containers and references. + +.. autodata:: DeserializeMode + +.. autoexception:: UnsafeDeserializationError diff --git a/examples/menudet.py b/examples/menudet.py new file mode 100644 index 000000000..6a5958031 --- /dev/null +++ b/examples/menudet.py @@ -0,0 +1,32 @@ +"""Menudet 1.0 draft: one kernel, array and independent table-row execution.""" + +from dataclasses import dataclass + +import numpy as np + +import blosc2 + +author = blosc2.DSLKernel.from_source("def affine(x):\n return x * 2.0 - 1.0\n") +kernel = blosc2.PortableKernel.from_json(author.export({"x": "float64"}, "float64")) +x = np.arange(6, dtype="float64") +values = kernel.lazy({"x": x}, partitions=(3,))[:] +np.testing.assert_array_equal(values, [-1, 1, 3, 5, 7, 9]) +print("Array:", values) + + +@dataclass +class Row: + amount: float = 0.0 + + +table = blosc2.CTable(Row, new_data={"amount": x.tolist()}, create_summary_index=False) +table.add_portable_computed_column("adjusted", kernel, inputs={"x": "amount"}, row_domain="independent") +np.testing.assert_array_equal(table["adjusted"][:], values) +print("Table:", table["adjusted"][:]) + +# A reduction is a different result contract: one scalar per ORIGINAL group. +total = blosc2.DSLKernel.from_source("def total(x):\n return sum(x)\n") +record = total.export({"x": "float64"}, "float64", cardinality="block_scalar") +totals = blosc2.PortableKernel.from_json(record).lazy({"x": x}, partitions=(3,))[:] +np.testing.assert_array_equal(totals, [3, 12]) +print("Block totals:", totals) diff --git a/plans/portable-DSL-1.0-implemenation.md b/plans/portable-DSL-1.0-implemenation.md index 50945054c..072fe2887 100644 --- a/plans/portable-DSL-1.0-implemenation.md +++ b/plans/portable-DSL-1.0-implemenation.md @@ -449,12 +449,15 @@ authoritative commands/results are in `portable-DSL-1.0-progress.md`. - [x] Compact local Python/native conformance and full local ASAN/UBSAN. - [x] Isolated native-development override wheels and artifact-disabled smoke. - [x] Draft/API docs build; changed portable page free of diagnostics. - - [ ] Windows/Linux/WASM external execution and independent per-function - accuracy certification (no universal ULP bound inferred from host libm). + - [x] Native Windows/Linux/macOS/WASM conformance matrix (all seven jobs green). + - [x] Independent finite math corpus and stock-pin macOS execution: 284 samples, + 40 canonical operations, both floating precisions, function-specific budgets. + - [ ] Execute the independent math corpus on remaining Python release platforms + (no universal ULP bound inferred from the finite corpus or host libm). - [x] Publish/pin the native revision and verify a stock-pin macOS ARM64 wheel. - [ ] Complete the remaining stock-pin platform wheel matrix. - - [ ] Clean whole-tree docs gate: existing installation/list-table, missing - image/generated-documentation/reference warnings outside portable changes. + - [x] Whole-tree docs disposition: fix parsing errors, exclude generated input + trees, and inventory/defer existing image/reference warnings outside Menudet. Each step must land as a usable tested increment; reduction and string fixtures can be prepared while numeric interpretation is implemented, without introducing diff --git a/plans/portable-DSL-1.0-progress.md b/plans/portable-DSL-1.0-progress.md index 210c24b13..38b271bb9 100644 --- a/plans/portable-DSL-1.0-progress.md +++ b/plans/portable-DSL-1.0-progress.md @@ -1,5 +1,142 @@ # Portable DSL 1.0 implementation progress +## Menudet / 4.15.0 release hardening — current handoff + +The user selected **Menudet — a little language for portable computation on +arrays and tables**, and approved the release-hardening plan. Menudet is the +portable language; miniexpr implements it. Existing API/format identifiers remain +unchanged. Broader JIT acceleration is deliberately separate: the native guard +still forces portable interpretation, preserving checked/ordered semantics. + +### Delivered in this increment + +- Both version fields now read `4.15.0.dev0` (development, not final publication). +- Python reference, release notes and native spec introductions name Menudet, + retain the draft warning, and distinguish full DSL from portable semantics. +- `examples/menudet.py` demonstrates the same array/table kernel plus explicit + logical block-scalar reductions; assertions passed against the installed wheel. +- Added `scripts/certify_menudet_math.py` and its checked-in JSON corpus: **284 + samples, 40 canonical functions, float32 and float64**, generated with mpmath + 1.3.0 at 100/200 decimal digits. Regeneration equality and deliberate failure + detection passed. No new runtime dependency; mpmath is needed only to regenerate. +- Function-specific finite sample budgets and coverage limitations are documented + in `doc/reference/menudet_accuracy.md`. These are not universal ULP guarantees + or certification of all possible inputs. Native exceptional/exact/alias fixtures + remain required. +- Wired the corpus into native Python, WASM and installed-wheel CI. Native jobs + upload platform reports; tested wheels also execute the seven portable modules. + +### Verification + +All Python/build commands used conda `blosc2`. Fresh stock build fetched native +`3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4`; both source override cache entries +are empty. Isolated imports asserted the installed target path and excluded the +editable finder only within the test subprocess. + +- Working-tree stock wheel: **11160 tests passed, 53 skips** (excluding heavy, + network and TUI), plus **120 installed-module doctests passed, 2 skips** in a + clean working directory. The serial combined invocation initially encountered + one relative-filename save failure: installed doctest modules are outside the + repository root fixture's isolation. All tests passed, and all doctests passed + on the clean-directory rerun; no user's existing file was removed to fix it. +- Independent corpus on that wheel: **284 passed**, maximum observed error + **3 ULP** (lgamma/float64); no tolerance changed after observing results. +- Built an **8,044,023-byte clean sdist**, verified inclusion of checker, corpus + and example, built a wheel from the unpacked sdist, and isolated-installed it. + This wheel passed **183 portable tests, 17 explicit external opt-in skips**, + plus all **284 independent math samples**. +- Clean-sdist wheel SHA256: + `c7fcc99d44fe12a7cdffc717eff4bc04450954c6d51eb98b282d09c2646b0d67`. +- Clean snapshot copied only tracked files and the four requested new files; + unrelated untracked local work was neither packaged nor deleted. A direct + dirty-checkout sdist attempt timed out and is not a release artifact. +- Ruff, format, workflow YAML, reference regeneration and whitespace checks pass. + +### Documentation disposition + +Fixed installation list-table indentation, stopped treating generated `doc/html` +and autosummary stash trees as sources, and documented NumPy's finfo alias without +reparsing its incompatible nested sections. Documented deserialization policy +types/errors and taught the public-API tripwire to recognize manual `py:class`. +Fresh whole-tree Sphinx now has **zero ERROR diagnostics**, and no Menudet page +diagnostic. It still has existing non-Menudet warnings: the fresh build reported +769 (639 autosummary stub, 100 duplicate object/label, 11 ambiguous references, +2 image, 17 other); the final follow-up with notebook execution disabled reports +772, with no undocumented-finfo warning. They remain visible, unsuppressed +documentation backlog, not a claim of a warnings-as-errors clean build. Menudet +pages and parser errors are the scope of this draft's documentation acceptance; +broad autosummary/reference cleanup is explicitly deferred. + +### External gates still pending + +The updated Python branch/workflows are not remotely available, so no new Python +platform job can certify these changes yet. Native miniexpr's seven-job matrix +is already green, but it does not certify Python artifacts/wheels. Windows, +Linux, Pyodide, free-threaded/ABI/platform wheel jobs must run the updated corpus +and integration tests and resolve any failures before releasing. No release or +beta publication occurred; final version metadata waits for those gates. + +Temporary directories under the approved temp root: `menudet-release-build`, +`menudet-release-wheel`, `menudet-release-installed`, `menudet-release-snapshot`, +`menudet-clean-sdist`, `menudet-sdist-unpacked`, `menudet-sdist-wheel`, +`menudet-sdist-installed`, `menudet-release-docs-clean`, and +`menudet-release-doctrees-clean`. Reports: `menudet-math-macos.json` and +`menudet-sdist-math-macos.json`. + +## Dependency pin updated after green native CI + +At the user's request, `CMakeLists.txt` now pins the green native revision +`3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4` from CI run `37612064807`. +Earlier notes saying the Python pin remains at `55c882b` are superseded. +Stock Python wheels have not yet been rebuilt/tested against this updated pin. + +## Native CI green (supersedes investigation below) + +At the user's request, published the fixes and monitored/repaired CI until green. +Revision `02791f338a853af517b6fedbd38c18d5a2b8da1a` fixed WASM and Windows +builtin recognition, but run `37611459450` exposed a second Windows issue: +identical-code folding merged the floating cast callback with real/conj identity +callbacks, misclassifying their output type. A unique volatile read in the cast +callback preserves its address identity without altering signed zero or NaNs. +The anchor fixture now reports the source and dtypes on a type mismatch. + +Final published revision: `3418cdce4b5c11e1661c8b6453e3d31d94b0b2b4`. +CI run [37612064807](https://github.com/Blosc/miniexpr/actions/runs/37612064807) +completed **successfully: all seven jobs green**, including Windows x64/ARM64, +Linux x64, Linux ARM64 interpreter/JIT, macOS, and WASM (including side-module +helper and JIT trace checks). Targeted local portable interpreter test also passed +after the cast-identity repair. Python's dependency pin remains at `55c882b`; +this entry does not claim Python stock-wheel certification of the final fixes. + +## Native CI failure investigation (2026-10-07) + +CI run `37610312891` on native revision `55c882b` failed on Windows x64, +Windows ARM64, and WASM. Both Windows failures occur while compiling the portable +`hypot(x, 2.0)` fixture: libm address recognition fails. WASM compilation fails +because its fenv lacks `FE_INVALID`; the new test also incorrectly assumed host +rounding/thread facilities. + +Local fixes in miniexpr: + +- `src/functions.c`: typed builtin recognition also matches the exact builtin + registration address, handling CRT inline/import aliases such as Windows hypot. +- `tests/test_dsl_portable_interp.c`: WASM retains numeric/literal/environment + checks under its fixed nearest rounding, without requiring unavailable exception + flags or pthreads. Native upward-rounding, exception and concurrency checks stay. + +Verification: complete native interpreter suite **325 passed**; local Emscripten +interpreter-only suite **48 passed**; CI-style WASM with JIT enabled **50 passed**; +targeted native portable interpreter ASAN/UBSAN **passed**. Existing duplicate +library and warning-enabled unused-function warnings remain; no WASM build warning +was observed. Remote Windows jobs still require revalidation with these fixes; +the reported failed CI run is not claimed green. Published pin is unchanged. + +Build directories: `/miniexpr-portable-1-interpreter`, +`/miniexpr-portable-1-wasm-fix`, and +`/miniexpr-portable-1-sanitized`. WASM was configured with `emcmake`, +Node as emulator, SLEEF OFF, then TCC JIT ON/bundled host TCC OFF/trace ON to match +the CI configuration. All commands used `conda run -n blosc2`. + ## Published dependency update — latest status The user authorized committing/pushing miniexpr and committing the Python diff --git a/pyproject.toml b/pyproject.toml index 07664b095..fe9d45647 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "rich", "threadpoolctl; platform_machine != 'wasm32'", ] -version = "4.14.2.dev0" +version = "4.15.0.dev0" [project.entry-points."array_api"] blosc2 = "blosc2" diff --git a/scripts/certify_menudet_math.py b/scripts/certify_menudet_math.py new file mode 100644 index 000000000..64c70db6d --- /dev/null +++ b/scripts/certify_menudet_math.py @@ -0,0 +1,174 @@ +"""Bounded, independent Menudet math certification (not a universal libm bound). + +Normal checks need only installed blosc2 and NumPy. Regeneration additionally +needs mpmath; references must agree at 100 and 200 decimal digits. Checked-in +vectors store exact input/output encodings, with per-function ULP budgets. +Run from any directory: python scripts/certify_menudet_math.py +""" + +import argparse +import json +import platform +from pathlib import Path + +import numpy as np + +DATA = Path(__file__).resolve().with_name("menudet_math_vectors.json") + + +def reference_cases(mp): + """Finite representative domains, including endpoints and cancellation.""" + unary = { + "acos": (-1, -0.75, 0.125, 0.875, 1), + "acosh": (1, 1.0009765625, 1.5, 8), + "asin": (-1, -0.75, 0.125, 0.875, 1), + "asinh": (-8, -0.125, 0.0009765625, 2), + "atan": (-8, -0.125, 0.0009765625, 2), + "atanh": (-0.875, -0.125, 0.0009765625, 0.875), + "cbrt": (-8, -0.125, 0.25, 2), + "cos": (-8, -0.125, 0.0009765625, 2), + "cosh": (-8, -0.125, 0.0009765625, 2), + "cospi": (-2, -0.75, 0.125, 0.5), + "erf": (-4, -0.125, 0.0009765625, 2), + "erfc": (-4, -0.125, 0.0009765625, 2), + "exp": (-8, -0.125, 0.0009765625, 2), + "exp10": (-8, -0.125, 0.0009765625, 2), + "exp2": (-8, -0.125, 0.0009765625, 2), + "expm1": (-8, -0.125, 0.0009765625, 2), + "lgamma": (0.125, 0.75, 1.5, 8), + "log": (0.0009765625, 0.75, 1.0009765625, 8), + "log10": (0.0009765625, 0.75, 1.0009765625, 8), + "log1p": (-0.875, -0.125, 0.0009765625, 8), + "log2": (0.0009765625, 0.75, 1.0009765625, 8), + "sin": (-8, -0.125, 0.0009765625, 2), + "sinh": (-8, -0.125, 0.0009765625, 2), + "sinpi": (-2, -0.75, 0.125, 0.5), + "sqrt": (0, 0.125, 1.0009765625, 8), + "tan": (-8, -0.125, 0.0009765625, 2), + "tanh": (-8, -0.125, 0.0009765625, 2), + "tgamma": (0.125, 0.75, 1.5, 8), + } + special = { + "cbrt": lambda x: mp.sign(x) * mp.root(abs(x), 3), + "cospi": mp.cospi, + "exp10": lambda x: mp.power(10, x), + "exp2": lambda x: mp.power(2, x), + "lgamma": lambda x: mp.log(abs(mp.gamma(x))), + "log10": lambda x: mp.log(x, 10), + "log2": lambda x: mp.log(x, 2), + "sinpi": mp.sinpi, + "tgamma": mp.gamma, + } + budgets = {"cospi": 8, "sinpi": 8, "exp10": 8, "lgamma": 8, "tgamma": 8} + for name, inputs in unary.items(): + function = special[name] if name in special else getattr(mp, name) + yield name, function, [(x,) for x in inputs], budgets.get(name, 4) + binary = { + "atan2": (mp.atan2, [(-2, -3), (0.125, -8), (2, 3)], 4), + "hypot": (lambda x, y: mp.sqrt(x * x + y * y), [(3, 4), (0.125, 8), (2, 3)], 4), + "logaddexp": (lambda x, y: mp.log(mp.exp(x) + mp.exp(y)), [(-8, -8), (-8, 8), (0.125, 0.25)], 8), + "pow": (mp.power, [(0.125, 1.5), (2, -0.75), (8, 0.125)], 4), + "fdim": (lambda x, y: max(x - y, mp.mpf(0)), [(8, 0.125), (0.125, 8)], 0), + "fmin": (min, [(-8, 0.125), (0.125, 8)], 0), + "fmax": (max, [(-8, 0.125), (0.125, 8)], 0), + "copysign": (lambda x, y: abs(x) * mp.sign(y), [(0.125, -8), (-8, 0.125)], 0), + "fmod": (lambda x, y: x - int(x / y) * y, [(-8.125, 2), (8.125, 2)], 0), + "remainder": (lambda x, y: x - mp.nint(x / y) * y, [(-7, 2), (7, 2)], 0), + "ldexp": (lambda x, y: x * mp.power(2, int(y)), [(0.125, -8), (1.5, 8)], 0), + "fma": (lambda x, y, z: x * y + z, [(1.0009765625, 0.9990234375, -1), (0.125, 8, -0.75)], 0), + } + for name, (function, inputs, budget) in binary.items(): + yield name, function, inputs, budget + + +def generate(): + import mpmath as mp + + cases = [] + for name, function, inputs, budget in reference_cases(mp): + for dtype in ("float32", "float64"): + for args in inputs: + typed = [float(np.asarray(x, dtype=dtype)) for x in args] + results = [] + for precision in (100, 200): + with mp.workdps(precision): + result = function(*(mp.mpf(x) for x in typed)) + results.append(float(np.asarray(float(result), dtype=dtype)).hex()) + if results[0] != results[1]: + raise ValueError(f"Unstable reference: {name} {dtype} {typed}") + cases.append( + { + "function": name, + "dtype": dtype, + "inputs": [x.hex() for x in typed], + "expected": results[0], + "max_ulp": budget, + } + ) + return { + "generator": f"mpmath {mp.__version__}, 100/200 decimal digits", + "scope": "Finite inputs only; not a global accuracy or bitwise portability guarantee", + "cases": cases, + } + + +def check(record): + import blosc2 + + failures = [] + maxima = {} + for case in record["cases"]: + name, dtype = case["function"], case["dtype"] + names = list("xyz"[: len(case["inputs"])]) + signature = dict.fromkeys(names, dtype) + if name == "ldexp": + signature["y"] = "int64" + source = f"def k({', '.join(names)}):\n return {name}({', '.join(names)})\n" + artifact = blosc2.DSLKernel.from_source(source).export(signature, dtype) + kernel = blosc2.PortableKernel.from_json(artifact, jit=False) + inputs = { + key: np.array([float.fromhex(value)], dtype=signature[key]) + for key, value in zip(names, case["inputs"], strict=True) + } + actual = float(kernel.evaluate(inputs)[0]) + expected = float.fromhex(case["expected"]) + # Use the smaller adjacent-format gap, including at powers of two. Python + # doubles represent all tested f32/f64 gaps/differences exactly enough here. + ref = np.asarray(expected, dtype=dtype) + upper = float(np.nextafter(ref, np.asarray(np.inf, dtype=dtype))) + lower = float(np.nextafter(ref, np.asarray(-np.inf, dtype=dtype))) + gap = min(upper - expected, expected - lower) + error = abs(actual - expected) / gap + key = f"{name}/{dtype}" + maxima[key] = max(maxima.get(key, 0), error) + if not np.isfinite(actual) or error > case["max_ulp"]: + failures.append({**case, "actual": actual.hex(), "error_ulp": error}) + return { + "package": blosc2.__file__, + "version": blosc2.__version__, + "platform": platform.platform(), + "reference": record["generator"], + "samples": len(record["cases"]), + "max_observed_ulp": maxima, + "failures": failures, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--generate", action="store_true", help="regenerate independent reference vectors") + parser.add_argument("--report", type=Path, help="write machine-readable certification report") + args = parser.parse_args() + if args.generate: + DATA.write_text(json.dumps(generate(), indent=2) + "\n") + return + report = check(json.loads(DATA.read_text())) + if args.report: + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + if report["failures"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/menudet_math_vectors.json b/scripts/menudet_math_vectors.json new file mode 100644 index 000000000..9b3a3abef --- /dev/null +++ b/scripts/menudet_math_vectors.json @@ -0,0 +1,2622 @@ +{ + "generator": "mpmath 1.3.0, 100/200 decimal digits", + "scope": "Finite inputs only; not a global accuracy or bitwise portability guarantee", + "cases": [ + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "0x1.921fb60000000p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "0x1.359d260000000p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.720a3a0000000p+0", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.02be9c0000000p-1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "0x1.921fb54442d18p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "0x1.359d26f93b6c3p+1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.720a392c1d955p+0", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.02be9ce0b87cdp-1", + "max_ulp": 4 + }, + { + "function": "acos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.6a025c0000000p-5", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.ecc2ca0000000p-1", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.62636e0000000p+1", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.6a025bf510baep-5", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.ecc2caec5160ap-1", + "max_ulp": 4 + }, + { + "function": "acosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.62636e408c33bp+1", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.921fb60000000p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.b235320000000p-1", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.00abe00000000p-3", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.10c0660000000p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x1.921fb60000000p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.921fb54442d18p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.b235315c680dcp-1", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.00abe0c129e1ep-3", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.10c066d3e6932p+0", + "max_ulp": 4 + }, + { + "function": "asin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+0" + ], + "expected": "0x1.921fb54442d18p+0", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.6363720000000p+1", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fead0c0000000p-4", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffa0000000p-11", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.7192180000000p+0", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.63637195fabd3p+1", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fead0b6996973p-4", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffaaaaad11p-11", + "max_ulp": 4 + }, + { + "function": "asinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.719218313d087p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.7249fa0000000p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd5baa0000000p-4", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff60000000p-11", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.1b6e1a0000000p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.7249faa996a21p+0", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd5ba9aac2f6ep-4", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff55555bbcp-11", + "max_ulp": 4 + }, + { + "function": "atan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.1b6e192ebbe44p+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.5aa1640000000p+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.0158920000000p-3", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000060000000p-10", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.5aa1640000000p+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.5aa16394d481fp+0", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.015891c9eaef7p-3", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000055555889p-10", + "max_ulp": 4 + }, + { + "function": "atanh", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p-1" + ], + "expected": "0x1.5aa16394d481fp+0", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p+1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-2" + ], + "expected": "0x1.428a300000000p-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.428a300000000p+0", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p+1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-2" + ], + "expected": "0x1.428a2f98d728bp-1", + "max_ulp": 4 + }, + { + "function": "cbrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.428a2f98d728bp+0", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.29fbec0000000p-3", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.fc01560000000p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff00000000p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.aa22660000000p-2", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.29fbebf632f94p-3", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.fc015527d5bd3p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff00000155p-1", + "max_ulp": 4 + }, + { + "function": "cos", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.aa22657537205p-2", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.749eaa0000000p+10", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.0200aa0000000p+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000080000000p+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.e18fa00000000p+1", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.749eaa93f4e76p+10", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.0200aac16db6fp+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.00000800000abp+0", + "max_ulp": 4 + }, + { + "function": "cosh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.e18fa0df2d9bcp+1", + "max_ulp": 4 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e60000000p-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.d906bc0000000p-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e667f3bcdp-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.d906bcf328d46p-1", + "max_ulp": 8 + }, + { + "function": "cospi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "-0x1.0000000000000p+0", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1f5e1a0000000p-3", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.20dd6e0000000p-10", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.fd9ae20000000p-1", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "-0x1.ffffff7b91176p-1", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1f5e1a35c3b89p-3", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.20dd6eff8cb8fp-10", + "max_ulp": 4 + }, + { + "function": "erf", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.fd9ae142795e3p-1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "0x1.0000000000000p+1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.23ebc40000000p+0", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ff6f920000000p-1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.328f5e0000000p-8", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+2" + ], + "expected": "0x1.ffffffbdc88bbp+0", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.23ebc346b8771p+0", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ff6f91488039ap-1", + "max_ulp": 4 + }, + { + "function": "erfc", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.328f5ec350e67p-8", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5fc2100000000p-12", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.c3d6a20000000p-1", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0040080000000p+0", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d8e64c0000000p+2", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5fc21041027adp-12", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.c3d6a24ed8222p-1", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.00400800aab55p+0", + "max_ulp": 4 + }, + { + "function": "exp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d8e64b8d4ddaep+2", + "max_ulp": 4 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5798ee0000000p-27", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.7ff2220000000p-1", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0093880000000p+0", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.9000000000000p+6", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.5798ee2308c3ap-27", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.7ff2224115d9ap-1", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.009388004be7ep+0", + "max_ulp": 8 + }, + { + "function": "exp10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.9000000000000p+6", + "max_ulp": 8 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-8", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.d5818e0000000p-1", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.002c600000000p+0", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+2", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-8", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "0x1.d5818dcfba487p-1", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.002c605e2e8cfp+0", + "max_ulp": 4 + }, + { + "function": "exp2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+2", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.ffd4080000000p-1", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.e14aee0000000p-4", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0020020000000p-10", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.98e64c0000000p+2", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.ffd407bdf7dfbp-1", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.e14aed893eef4p-4", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.002002aad5577p-10", + "max_ulp": 4 + }, + { + "function": "expm1", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.98e64b8d4ddaep+2", + "max_ulp": 4 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.027c4c0000000p+1", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.a051c40000000p-3", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "-0x1.eeb95c0000000p-4", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.10ce200000000p+3", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.027c4cfd515b0p+1", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.a051c372609eep-3", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "-0x1.eeb95b094c191p-4", + "max_ulp": 8 + }, + { + "function": "lgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.10ce1f32dcc30p+3", + "max_ulp": 8 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.bb9d3c0000000p+2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.2696220000000p-2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.ffc00a0000000p-11", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.0a2b240000000p+1", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.bb9d3beb8c86bp+2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.269621134db92p-2", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.ffc00aa8ab110p-11", + "max_ulp": 4 + }, + { + "function": "log", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.0a2b23f3bab73p+1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.8151820000000p+1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.ffbfc20000000p-4", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.bc80240000000p-12", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.ce61d00000000p-1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.8151824c7587fp+1", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.ffbfc2bbc7803p-4", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.bc80239e5b560p-12", + "max_ulp": 4 + }, + { + "function": "log10", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.ce61cf8ef36fep-1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.0a2b240000000p+1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1178e80000000p-3", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ffc00a0000000p-11", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.193ea80000000p+1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "-0x1.c000000000000p-1" + ], + "expected": "-0x1.0a2b23f3bab73p+1", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.1178e8227e47cp-3", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.ffc00aa8ab110p-11", + "max_ulp": 4 + }, + { + "function": "log1p", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.193ea7aad030bp+1", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.4000000000000p+3", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.a8ff980000000p-2", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.7126540000000p-10", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+1", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "-0x1.4000000000000p+3", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "-0x1.a8ff971810a5ep-2", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.712653743f454p-10", + "max_ulp": 4 + }, + { + "function": "log2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fa8d2a0000000p-1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.feaaee0000000p-4", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffa0000000p-11", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d18f6e0000000p-1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fa8d2a028cf7bp-1", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.feaaeee86ee36p-4", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffffaaaaaaefp-11", + "max_ulp": 4 + }, + { + "function": "sin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d18f6ead1b446p-1", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.749ea60000000p+10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.00aacc0000000p-3", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000020000000p-10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d03cf60000000p+1", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.749ea514eca66p+10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.00aaccd00d2f1p-3", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.000002aaaaacdp-10", + "max_ulp": 4 + }, + { + "function": "sinh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.d03cf63b6e19fp+1", + "max_ulp": 4 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e60000000p-1", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.87de2a0000000p-2", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+1" + ], + "expected": "0x0.0p+0", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "-0x1.8000000000000p-1" + ], + "expected": "-0x1.6a09e667f3bcdp-1", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.87de2a6aea963p-2", + "max_ulp": 8 + }, + { + "function": "sinpi", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 8 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x0.0p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.6a09e60000000p-2", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.001ffe0000000p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.6a09e60000000p+1", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x0.0p+0" + ], + "expected": "0x0.0p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.6a09e667f3bcdp-2", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0" + ], + "expected": "0x1.001ffe003ff60p+0", + "max_ulp": 4 + }, + { + "function": "sqrt", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.6a09e667f3bcdp+1", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.b32e780000000p+2", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.01577a0000000p-3", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000060000000p-10", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.17af620000000p+1", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "0x1.b32e78f49a1e4p+2", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.01577af1511a5p-3", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.0000055555777p-10", + "max_ulp": 4 + }, + { + "function": "tan", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "-0x1.17af62e0950f8p+1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fffff80000000p-1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd59920000000p-4", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff60000000p-11", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.ed95060000000p-1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.fffff872a91f8p-1", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p-3" + ], + "expected": "-0x1.fd5992bc4b835p-4", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-10" + ], + "expected": "0x1.fffff5555599ap-11", + "max_ulp": 4 + }, + { + "function": "tanh", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1" + ], + "expected": "0x1.ed9505e1bc3d4p-1", + "max_ulp": 4 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.e22c1a0000000p+2", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.39b4e80000000p+0", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.c5bf8a0000000p-1", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.3b00000000000p+12", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3" + ], + "expected": "0x1.e22c196233d21p+2", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p-1" + ], + "expected": "0x1.39b4e8b50f62cp+0", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0" + ], + "expected": "0x1.c5bf891b4ef6bp-1", + "max_ulp": 8 + }, + { + "function": "tgamma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3" + ], + "expected": "0x1.3b00000000000p+12", + "max_ulp": 8 + }, + { + "function": "atan2", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+1", + "-0x1.8000000000000p+1" + ], + "expected": "-0x1.46dc0a0000000p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.901fc00000000p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.2d0eae0000000p-1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+1", + "-0x1.8000000000000p+1" + ], + "expected": "-0x1.46dc09ec29433p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.901fbfee871a6p+1", + "max_ulp": 4 + }, + { + "function": "atan2", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.2d0ead6066395p-1", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+1", + "0x1.0000000000000p+2" + ], + "expected": "0x1.4000000000000p+2", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0008000000000p+3", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.cd82b40000000p+1", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+1", + "0x1.0000000000000p+2" + ], + "expected": "0x1.4000000000000p+2", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0007ffe000fffp+3", + "max_ulp": 4 + }, + { + "function": "hypot", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1", + "0x1.8000000000000p+1" + ], + "expected": "0x1.cd82b446159f3p+1", + "max_ulp": 4 + }, + { + "function": "logaddexp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.d3a37a0000000p+2", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p-2" + ], + "expected": "0x1.c3e4060000000p-1", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.d3a37a020b8c2p+2", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000003c6ab74p+3", + "max_ulp": 8 + }, + { + "function": "logaddexp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p-2" + ], + "expected": "0x1.c3e4055056374p-1", + "max_ulp": 8 + }, + { + "function": "pow", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.8000000000000p+0" + ], + "expected": "0x1.6a09e60000000p-5", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+1", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.306fe00000000p-1", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.4bfdae0000000p+0", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.8000000000000p+0" + ], + "expected": "0x1.6a09e667f3bcdp-5", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+1", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.306fe0a31b715p-1", + "max_ulp": 4 + }, + { + "function": "pow", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.4bfdad5362a27p+0", + "max_ulp": 4 + }, + { + "function": "fdim", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.f800000000000p+2", + "max_ulp": 0 + }, + { + "function": "fdim", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x0.0p+0", + "max_ulp": 0 + }, + { + "function": "fdim", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.f800000000000p+2", + "max_ulp": 0 + }, + { + "function": "fdim", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x0.0p+0", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "-0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmin", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmax", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float32", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "copysign", + "dtype": "float64", + "inputs": [ + "-0x1.0000000000000p+3", + "0x1.0000000000000p-3" + ], + "expected": "0x1.0000000000000p+3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float32", + "inputs": [ + "-0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float32", + "inputs": [ + "0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float64", + "inputs": [ + "-0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "fmod", + "dtype": "float64", + "inputs": [ + "0x1.0400000000000p+3", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p-3", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float32", + "inputs": [ + "-0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float32", + "inputs": [ + "0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float64", + "inputs": [ + "-0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "remainder", + "dtype": "float64", + "inputs": [ + "0x1.c000000000000p+2", + "0x1.0000000000000p+1" + ], + "expected": "-0x1.0000000000000p+0", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-11", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float32", + "inputs": [ + "0x1.8000000000000p+0", + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+8", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "-0x1.0000000000000p+3" + ], + "expected": "0x1.0000000000000p-11", + "max_ulp": 0 + }, + { + "function": "ldexp", + "dtype": "float64", + "inputs": [ + "0x1.8000000000000p+0", + "0x1.0000000000000p+3" + ], + "expected": "0x1.8000000000000p+8", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float32", + "inputs": [ + "0x1.0040000000000p+0", + "0x1.ff80000000000p-1", + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.0000000000000p-20", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float32", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.0000000000000p-2", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float64", + "inputs": [ + "0x1.0040000000000p+0", + "0x1.ff80000000000p-1", + "-0x1.0000000000000p+0" + ], + "expected": "-0x1.0000000000000p-20", + "max_ulp": 0 + }, + { + "function": "fma", + "dtype": "float64", + "inputs": [ + "0x1.0000000000000p-3", + "0x1.0000000000000p+3", + "-0x1.8000000000000p-1" + ], + "expected": "0x1.0000000000000p-2", + "max_ulp": 0 + } + ] +} diff --git a/src/blosc2/version.py b/src/blosc2/version.py index 241d027ca..7e3c7039c 100644 --- a/src/blosc2/version.py +++ b/src/blosc2/version.py @@ -1,2 +1,2 @@ -__version__ = "4.14.2.dev0" +__version__ = "4.15.0.dev0" __array_api_version__ = "2024.12" From e0e0f3bb25d75a36af719fe118a08420f5f52dc3 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 7 Oct 2026 14:03:15 +0200 Subject: [PATCH 26/30] Allow pre-release wheel matrix certification --- .github/workflows/cibuildwheels.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml index 99648a9d4..fde5ef245 100644 --- a/.github/workflows/cibuildwheels.yml +++ b/.github/workflows/cibuildwheels.yml @@ -1,6 +1,8 @@ name: Python wheels on: + # Certify development revisions without creating a release tag or publishing. + workflow_dispatch: # Trigger the workflow only for tags and PRs to the main branch push: tags: @@ -168,7 +170,9 @@ jobs: CIBW_BUILD: ${{ matrix.cibw_build }} # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF" - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: >- + pytest {project}/tests && + python {project}/scripts/certify_menudet_math.py # Pin the Pyodide version explicitly per target for reproducible builds. # Note: in-memory SChunk get_slice is broken on the Pyodide 0.29.x # Emscripten toolchain (works on 314); see gh-664. The affected test is @@ -294,6 +298,9 @@ jobs: python -m pytest tests/test_open.py tests/test_vlmeta.py \ tests/ndarray/test_evaluate.py -q -p no:cacheprovider + - name: Certify Menudet on the installed abi3 wheel + run: python scripts/certify_menudet_math.py + upload_pypi: needs: [ build_wheels, build_wheels_wasm, test_abi3_matrix ] From df9b5a39f8651576c912e1463ebc7776838c190c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 7 Oct 2026 14:18:14 +0200 Subject: [PATCH 27/30] Allow safe GIL fallback during free-threaded wheel tests --- .github/workflows/cibuildwheels.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml index fde5ef245..d5ddafab7 100644 --- a/.github/workflows/cibuildwheels.yml +++ b/.github/workflows/cibuildwheels.yml @@ -15,6 +15,8 @@ env: CIBW_BUILD_VERBOSITY: 1 # Every tested wheel checks portable descriptors/persistence and independent # numeric references, not only whether the extension can be imported. + # Free-threaded CPython safely enables the GIL for this extension. Allow only + # that expected import warning; never force PYTHON_GIL=0 or claim no-GIL safety. CIBW_TEST_COMMAND: >- python -m pytest {project}/tests/test_portable_artifact.py @@ -23,7 +25,8 @@ env: {project}/tests/test_portable_lazy.py {project}/tests/test_portable_policy.py {project}/tests/test_portable_table_export.py - {project}/tests/test_dsl_portable.py -q -n 0 && + {project}/tests/test_dsl_portable.py -q -n 0 + -W "ignore:The global interpreter lock .*has been enabled to load module 'blosc2[.]blosc2_ext':RuntimeWarning" && python {project}/scripts/certify_menudet_math.py # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long # Now that github provides native arm64 runners, we can enable tests again From e768bc538cb096c7f5c669046ed3aed5e55d9c6b Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 7 Oct 2026 14:37:58 +0200 Subject: [PATCH 28/30] Apply the scoped GIL warning regex through pytest configuration --- .github/workflows/cibuildwheels.yml | 3 +-- pytest.ini | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml index d5ddafab7..14be8aa4b 100644 --- a/.github/workflows/cibuildwheels.yml +++ b/.github/workflows/cibuildwheels.yml @@ -25,8 +25,7 @@ env: {project}/tests/test_portable_lazy.py {project}/tests/test_portable_policy.py {project}/tests/test_portable_table_export.py - {project}/tests/test_dsl_portable.py -q -n 0 - -W "ignore:The global interpreter lock .*has been enabled to load module 'blosc2[.]blosc2_ext':RuntimeWarning" && + {project}/tests/test_dsl_portable.py -q -n 0 && python {project}/scripts/certify_menudet_math.py # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long # Now that github provides native arm64 runners, we can enable tests again diff --git a/pytest.ini b/pytest.ini index f566db577..d0a61ad55 100644 --- a/pytest.ini +++ b/pytest.ini @@ -38,3 +38,7 @@ markers = filterwarnings = error ignore::UserWarning + # Free-threaded CPython safely enables the GIL for this extension. This is + # an expected import notification, not a claim that no-GIL execution is safe. + # Keep the regex here: pytest's command-line -W treats the message literally. + ignore:The global interpreter lock \(GIL\) has been enabled to load module 'blosc2\.blosc2_ext':RuntimeWarning From fd672723e4cebe7cf91618f089ac8a11b5ed22d8 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 7 Oct 2026 15:03:22 +0200 Subject: [PATCH 29/30] Record green Menudet release certification --- plans/portable-DSL-1.0-implemenation.md | 7 ++-- plans/portable-DSL-1.0-progress.md | 49 +++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/plans/portable-DSL-1.0-implemenation.md b/plans/portable-DSL-1.0-implemenation.md index 072fe2887..a01531f31 100644 --- a/plans/portable-DSL-1.0-implemenation.md +++ b/plans/portable-DSL-1.0-implemenation.md @@ -445,17 +445,18 @@ authoritative commands/results are in `portable-DSL-1.0-progress.md`. and reject persistence with a specific ambiguity diagnostic. 9. [x] Conservative interpreter-only dispatch and pre-execution optional JIT fallback; required-JIT rejects unsupported acceleration explicitly. -10. [ ] Complete release verification gates: +10. [x] Complete release verification gates for the tested platform matrix: - [x] Compact local Python/native conformance and full local ASAN/UBSAN. - [x] Isolated native-development override wheels and artifact-disabled smoke. - [x] Draft/API docs build; changed portable page free of diagnostics. - [x] Native Windows/Linux/macOS/WASM conformance matrix (all seven jobs green). - [x] Independent finite math corpus and stock-pin macOS execution: 284 samples, 40 canonical operations, both floating precisions, function-specific budgets. - - [ ] Execute the independent math corpus on remaining Python release platforms + - [x] Execute the independent math corpus on remaining Python release platforms (no universal ULP bound inferred from the finite corpus or host libm). - [x] Publish/pin the native revision and verify a stock-pin macOS ARM64 wheel. - - [ ] Complete the remaining stock-pin platform wheel matrix. + - [x] Complete the remaining stock-pin platform wheel matrix (27 successful jobs; + retain the existing untested macOS Intel cp315/cp315t dependency exception). - [x] Whole-tree docs disposition: fix parsing errors, exclude generated input trees, and inventory/defer existing image/reference warnings outside Menudet. diff --git a/plans/portable-DSL-1.0-progress.md b/plans/portable-DSL-1.0-progress.md index 38b271bb9..7c4a48ecd 100644 --- a/plans/portable-DSL-1.0-progress.md +++ b/plans/portable-DSL-1.0-progress.md @@ -1,5 +1,54 @@ # Portable DSL 1.0 implementation progress +## Published release certification — green platform matrix + +The user authorized publishing changes and monitoring/fixing CI until green. +Native Menudet documentation was published as miniexpr +`36770f87e10f1b92bfe0f9f92a7039eba2e976ea`; all seven native jobs passed in +[37618199852](https://github.com/Blosc/miniexpr/actions/runs/37618199852). +Python still pins the already-green runtime revision `3418cdce...`: the subsequent +native changes are documentation/matrix metadata only, not runtime changes. + +Python release-hardening and certification fixes are published on `portable-dsls`. +Certified implementation/workflow revision: +`e768bc538cb096c7f5c669046ed3aed5e55d9c6b`. Successful runs: + +- [Tests 37622410469](https://github.com/Blosc/python-blosc2/actions/runs/37622410469) + — all standard platform jobs green, including independent math reports. +- [WASM 37622410514](https://github.com/Blosc/python-blosc2/actions/runs/37622410514) + — Python/Pyodide suite plus the independent corpus green. +- [Python wheels 37622420247](https://github.com/Blosc/python-blosc2/actions/runs/37622420247) + — **27 successful jobs**: seven native/WASM build jobs and twenty abi3 test + jobs across Python 3.11–3.15. Includes Linux x64/ARM64, Windows x64/ARM64, + macOS x64/ARM64, Pyodide 3.13/3.14 and free-threaded builds. PyPI upload is + intentionally skipped (no tag/publication). The pre-existing skip of testing + macOS Intel cp315/cp315t wheels is retained for unavailable pre-release dependency + wheels; do not claim those specific wheels were tested. + +The first two wheel runs (`37618308296`, `37620050340`) failed because importing +the extension safely enables the GIL on free-threaded Python, emitting a +RuntimeWarning that pytest promoted to an error. A command-line filter initially +failed because pytest treats CLI warning messages literally, not as regexes. +Moved the **exact extension-specific regex** into `pytest.ini`; other runtime +warnings remain errors. A local synthetic check confirms both acceptance of the +expected import warning and rejection of an unrelated numeric RuntimeWarning. +No `PYTHON_GIL=0`, no no-GIL safety declaration, no tolerance changes or new +conformance skips were introduced. + +Added `workflow_dispatch` to the wheel workflow to certify development revisions +without release tags. Both wheel Pyodide versions and each abi3 matrix job now +also run the independent corpus. + +Downloaded and verified all five native Python math reports: every report has +**284 samples and zero failures**. Maximum observed ULP: Windows **2**, Linux +**2** (including NumPy 1.26 and Python 3.14 jobs), macOS ARM64 **3**. Reports are +under `/menudet-native-ci-math-reports`. Wheel/WASM corpus executions +are additionally certified by the successful corresponding job commands. + +The earlier external-platform/published-branch blockers below are superseded. +Release certification for the tested matrix is now green; final release version, +publication, beta feedback and compatibility freeze remain separate decisions. + ## Menudet / 4.15.0 release hardening — current handoff The user selected **Menudet — a little language for portable computation on From 62135e15e9945c93941a4ee2f348f6920481b053 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 8 Oct 2026 08:19:04 +0200 Subject: [PATCH 30/30] Unify Menudet table kernels and restore LazyExpr persistence --- RELEASE_NOTES.md | 17 +- doc/reference/portable_dsl.rst | 28 +-- doc/reference/save_load.rst | 6 + examples/menudet.py | 2 +- plans/portable-DSL-1.0-progress.md | 35 +++ src/blosc2/b2objects.py | 21 -- src/blosc2/ctable.py | 288 ++++++++-------------- src/blosc2/exceptions.py | 10 +- tests/ctable/test_ctable_computed_cols.py | 27 +- tests/ctable/test_ctable_dsl_columns.py | 25 +- tests/ndarray/test_lazyexpr.py | 63 ++--- tests/test_deserialization.py | 11 + tests/test_portable_policy.py | 56 ++++- tests/test_portable_table_export.py | 66 ++++- 14 files changed, 364 insertions(+), 291 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index a91a5186f..7bcdb6756 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -13,12 +13,25 @@ flow, fixed-width strings, logical array coordinates and ordered block reductions. - Keep logical reduction groups stable through persistence, partial reads and output rechunking. True block-scalar results expose the logical block-grid shape. -- Add portable CTable computed/generated transformers with an explicit - `row_domain="independent"` contract, including fixed-shape row reductions. +- Validate kernel inputs to `CTable.add_computed_column()` and + `add_generated_column()` as Menudet artifacts with implicit independent-row + semantics, including fixed-shape row reductions. Named bindings are supported; + positional bindings and column-bound DSL LazyUDFs adopt the same semantics. + New cross-row computations must use array partitions or table aggregation, + rather than implicit execution blocks or append batches. +- Remove the draft `add_portable_*` column methods and `row_domain` argument. + Fixed-shape row reductions use `cardinality="block_scalar"`. - New compliant DSLKernel-backed LazyUDF saves export the draft automatically; unsupported saves fail validation before writing the destination. - Historical Python-specific recipes require explicit `deserialize="full"`. MessagePack decoding is safe by default, including nested containers. +- Saved legacy table-kernel recipes retain their old block/batch-dependent + behavior only with trusted, explicit `deserialize="full"`; safe-mode errors + explain the opt-in. Recipes are never automatically reinterpreted. +- Ordinary LazyExpr persistence retains supported numerical calls, reductions, + methods and indexing under default loading, with existing expression validation. + Full-only nested payloads remain gated. String/lazy expressions and non-kernel + row transformers retain their existing table registration behavior. Menudet 1.0 is a **draft**, not a frozen compatibility promise. The existing `DSLKernel` and portable API names remain unchanged. This draft uses the typed diff --git a/doc/reference/portable_dsl.rst b/doc/reference/portable_dsl.rst index a2c496286..bd9d1035d 100644 --- a/doc/reference/portable_dsl.rst +++ b/doc/reference/portable_dsl.rst @@ -150,7 +150,7 @@ that policy propagates through nested references. Loaded legacy recipes are not automatically migrated by saving. CTable legacy computed/generated DSL metadata has the same explicit policy gate. The older table vector-column DSL API lacks an explicit row-domain contract and cannot be silently converted to independent -rows. Use the authored row API below instead. +rows when loading. New registrations use the independent-row contract below. ``msgpack_unpackb()`` also defaults to safe decoding. Active reconstruction needs explicit ``deserialize='full'``; container readers propagate their effective policy through nested values instead of relying on the helper's default. @@ -160,26 +160,26 @@ Draft native table transformers Scalar stored columns bind to native transformers by name:: - table.add_portable_computed_column("total", kernel, inputs={"x": "amount"}, - row_domain="independent") - table.add_portable_generated_column("stored_total", kernel, - inputs={"x": "amount"}, - row_domain="independent") + table.add_computed_column("total", kernel, inputs={"x": "amount"}) + table.add_generated_column("stored_total", values=kernel, + inputs={"x": "amount"}) -These APIs also accept a newly authored ``DSLKernel`` directly, with explicit -output dtype. Normalization, capture snapshots and native validation finish +These APIs also accept a newly authored ``DSLKernel`` directly. Specify the +output dtype when it differs from input type promotion. Normalization, capture snapshots and native validation finish before column registration or destination mutation:: - table.add_portable_computed_column( + table.add_computed_column( "twice", blosc2.DSLKernel.from_source("def twice(x):\n return x * 2\n"), - inputs={"x": "amount"}, dtype="int64", row_domain="independent") + inputs={"x": "amount"}, dtype="int64") For a fixed-shape row reduction, additionally specify -``cardinality="block_scalar"``. This row-domain declaration is required; no -grouping is inferred from the old vector-column API, table storage tiles or -append batches. +``cardinality="block_scalar"``. Independent-row grouping is the contract of +kernel registration; there is no ``row_domain`` option or separate +``add_portable_*`` API. Positional input lists and column-bound DSL LazyUDFs +also adopt this contract on new registration. String/lazy expressions and +non-kernel row transformers retain their existing behavior. -The explicit ``independent`` domain means **each scalar row is one original +The independent domain means **each scalar row is one original one-lane group**, not a storage block or a changing table-wide reduction. Rank-one coordinates are row-local: shape ``(1,)``, origin ``(0,)``, ``_i0 == 0`` and ``_n0 == 1``. Block-scalar and elementwise kernels both yield one scalar per row. diff --git a/doc/reference/save_load.rst b/doc/reference/save_load.rst index 16304a1c1..ba4e316d4 100644 --- a/doc/reference/save_load.rst +++ b/doc/reference/save_load.rst @@ -28,6 +28,12 @@ Deserialization policy Validated Menudet artifacts load natively without reconstructing Python code. Historical Python-specific recipes require explicit ``deserialize="full"``; the selected policy propagates through nested containers and references. +Ordinary LazyExpr expressions, including supported numerical calls, reductions, +methods and indexing, load under the default ``deserialize="safe"`` policy +with the existing expression validation. Full-only nested operands (such as +legacy kernels or proxies) still require explicit opt-in. Legacy table-kernel +recipes loaded with ``"full"`` retain their old block/batch-dependent grouping; +new registrations use independent-row Menudet semantics. .. autodata:: DeserializeMode diff --git a/examples/menudet.py b/examples/menudet.py index 6a5958031..a44debacd 100644 --- a/examples/menudet.py +++ b/examples/menudet.py @@ -20,7 +20,7 @@ class Row: table = blosc2.CTable(Row, new_data={"amount": x.tolist()}, create_summary_index=False) -table.add_portable_computed_column("adjusted", kernel, inputs={"x": "amount"}, row_domain="independent") +table.add_computed_column("adjusted", kernel, inputs={"x": "amount"}) np.testing.assert_array_equal(table["adjusted"][:], values) print("Table:", table["adjusted"][:]) diff --git a/plans/portable-DSL-1.0-progress.md b/plans/portable-DSL-1.0-progress.md index 7c4a48ecd..42a46092f 100644 --- a/plans/portable-DSL-1.0-progress.md +++ b/plans/portable-DSL-1.0-progress.md @@ -1,5 +1,40 @@ # Portable DSL 1.0 implementation progress +## Table API and LazyExpr compatibility follow-up + +The user approved unifying table kernel registration and restoring ordinary +LazyExpr persistence, then explicitly requested **no `add_portable_*` aliases**. +The final policy retains trusted legacy execution through explicit +`deserialize="full"`; it supersedes the briefly proposed unconditional rejection. + +- `add_computed_column()` / `add_generated_column()` now export and validate + authored kernels and accept native artifacts with named bindings. Positional + lists and column-bound DSL LazyUDFs adopt the same independent-row contract. + `assign()` uses the same normalization; new registrations have no vector-column + fallback. String/lazy expressions and non-kernel row transformers are unchanged. +- Removed the separate portable-column methods and public `row_domain` argument. + New metadata omits `row_domain`; prior draft metadata with the independent + marker still validates, and unknown grouping markers reject. Fixed-shape row + reductions use `cardinality="block_scalar"`. +- Safe loading rejects legacy table-column recipes before source reconstruction, + with an explicit message naming block/batch-dependent behavior and the trusted + `"full"` opt-in. Full loading preserves legacy execution, not reinterpretation. +- Removed the new arithmetic-only LazyExpr AST restriction. Existing expression + validation remains. Ordinary numerical-expression round-trip tests now reopen + with the default policy; nested legacy/proxy/other full-only payloads stay gated. +- Added default round trips for `sqrt(x)`, `sum(x)`, `x.mean()` and `x[0]`, row-local + reduction tests, persisted legacy computed/generated opt-in tests, and artifact + registration/no-mutation checks. Updated API docs, release notes and example. + +Verification and final CI run links are recorded below as they complete. Earlier +sections describe the original API and are historical where they differ here. + +Local verification: the full default suite (including module doctests) passed +**11,291 cases**, with 55 existing skips. All 284 independent math samples passed, the Menudet +example ran, and Ruff/format/whitespace checks passed. Sphinx completed with no +ERROR diagnostics or warnings on the modified Menudet/persistence pages; the +previous unrelated whole-tree warnings remain outside this change. + ## Published release certification — green platform matrix The user authorized publishing changes and monitoring/fixing CI until green. diff --git a/src/blosc2/b2objects.py b/src/blosc2/b2objects.py index a447c1051..d49a617ff 100644 --- a/src/blosc2/b2objects.py +++ b/src/blosc2/b2objects.py @@ -7,7 +7,6 @@ from __future__ import annotations -import ast import pathlib from typing import Any @@ -161,26 +160,6 @@ def decode_structured_lazyexpr(payload, *, carrier_path=None, deserialize="safe" from blosc2.lazyexpr import validate_expr validate_expr(expression) - if normalize_deserialize(deserialize) is DeserializeMode.SAFE: - allowed = ( - ast.Expression, - ast.BinOp, - ast.UnaryOp, - ast.BoolOp, - ast.Compare, - ast.Name, - ast.Load, - ast.Constant, - ast.operator, - ast.unaryop, - ast.boolop, - ast.cmpop, - ) - if any( - not isinstance(node, allowed) or (isinstance(node, ast.Name) and node.id not in operands_payload) - for node in ast.walk(ast.parse(expression, mode="eval")) - ): - raise UnsafeDeserializationError("non-arithmetic lazy expression") operands, missing_ops = decode_operand_mapping( operands_payload, base_path=carrier_path, deserialize=deserialize ) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 5ccb0d166..577c8d0df 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -12169,7 +12169,7 @@ def _readable_computed_expr(cc: dict) -> str: """ col_deps = cc["col_deps"] if cc.get("kind") == "portable": - return f"portable[{cc['row_domain']}]({', '.join(col_deps)})" + return f"portable[independent]({', '.join(col_deps)})" if cc.get("kind") == "dsl": kernel = cc.get("kernel") kname = getattr(kernel, "__name__", "dsl_kernel") @@ -12271,7 +12271,6 @@ def _schema_dict_with_computed(self) -> dict: "kind": "portable", "artifact": cc["artifact"], "bindings": cc["bindings"], - "row_domain": cc["row_domain"], "row_shape": list(cc["row_shape"]), "col_deps": cc["col_deps"], "dtype": str(cc["dtype"]), @@ -12314,9 +12313,7 @@ def _schema_dict_with_computed(self) -> dict: if "transformer" in meta: entry["transformer"] = meta["transformer"] if meta.get("transformer_kind") == "portable": - entry.update( - {key: meta[key] for key in ("artifact", "bindings", "row_domain", "row_shape")} - ) + entry.update({key: meta[key] for key in ("artifact", "bindings", "row_shape")}) if meta.get("dsl_source") is not None: entry["dsl_source"] = meta["dsl_source"] if meta.get("jit_backend") is not None: @@ -12562,9 +12559,7 @@ def _load_materialized_cols_from_schema(self, schema_dict: dict) -> None: desc = self._validate_portable_table_metadata(meta) if self._cols[meta["name"]].dtype != desc["dtype"]: raise ValueError("Stored portable output dtype disagrees with native signature") - loaded.update( - {key: desc[key] for key in ("artifact", "bindings", "row_domain", "row_shape", "kernel")} - ) + loaded.update({key: desc[key] for key in ("artifact", "bindings", "row_shape", "kernel")}) if meta.get("dsl_source") is not None: loaded["dsl_source"] = meta["dsl_source"] if meta.get("jit_backend") is not None: @@ -12588,7 +12583,12 @@ def _check_legacy_dsl_policy(self, schema_dict): getattr(getattr(storage, "_store", None), "_deserialize_mode", "safe"), ) if legacy and normalize_deserialize(policy) is not DeserializeMode.FULL: - raise UnsafeDeserializationError("legacy DSL table columns") + raise UnsafeDeserializationError( + "legacy DSL table columns with block/batch-dependent evaluation", + location="CTable column recipes", + hint="Reopen with deserialize='full' only if you trust this data and " + "intend to retain legacy table-kernel behavior.", + ) def _preflight_portable_persistence(self): for meta in [*self._computed_cols.values(), *self._materialized_cols.values()]: @@ -12600,8 +12600,8 @@ def _preflight_portable_persistence(self): ): raise TypeError( "Source-authored vector table columns have no explicit row-domain contract; " - "use add_portable_computed_column or add_portable_generated_column with " - "the DSLKernel, dtype=..., and row_domain='independent'. " + "use add_computed_column or add_generated_column with " + "the DSLKernel, named inputs, and dtype=.... " "Legacy source metadata is not automatically migrated." ) @@ -12964,26 +12964,6 @@ def _dsl_deps_from_lazyudf(self, lazyudf) -> list[str]: col_deps.append(cname) return col_deps - def _resolve_dsl_kernel(self, kernel, inputs) -> tuple[Any, list[str]]: - """Validate a bare DSL kernel + its ``inputs`` column bindings.""" - if kernel.dsl_error is not None: - raise blosc2.DSLSyntaxError(f"Invalid DSL kernel: {kernel.dsl_error}") - if inputs is None: - raise TypeError( - "A DSL kernel passed directly requires inputs=[...] naming one source " - "column per kernel parameter." - ) - col_deps = list(inputs) - expected = kernel.input_names - if expected is not None and len(col_deps) != len(expected): - raise ValueError( - f"DSL kernel expects {len(expected)} input(s) {expected}, " - f"but inputs={col_deps} provides {len(col_deps)}." - ) - for d in col_deps: - self._validate_transformer_dep(d) - return kernel, col_deps - def _guard_utf8_kernel_deps(self, col_deps) -> None: """Refuse a UDF/DSL kernel that reads a utf8 column, naming the column. @@ -13009,30 +12989,39 @@ def _guard_utf8_kernel_deps(self, col_deps) -> None: ) ) - def _normalize_transformer(self, expr, inputs=None) -> dict: + def _normalize_transformer(self, expr, inputs=None, *, dtype=None, cardinality="elementwise") -> dict: """Resolve *expr* into a transformer descriptor. Returns one of:: {"kind": "expression", "lazy": , "col_deps": [...]} - {"kind": "dsl", "kernel": , "col_deps": [...]} + {"kind": "portable", "kernel": , "bindings": {...}, ...} A ``@blosc2.dsl_kernel`` is accepted either directly (with *inputs* naming one source column per kernel parameter) or as a ``LazyUDF`` returned by a callable (then operands are matched to columns by identity and *inputs* is ignored). + New kernel registrations always validate/export Menudet and evaluate + independent rows; only loaded legacy recipes use vector-column execution. """ - if isinstance(expr, blosc2.DSLKernel): - if not isinstance(self._storage, InMemoryTableStorage): - raise TypeError("Persisting full DSL table columns is unsupported; use portable artifacts") - kernel, col_deps = self._resolve_dsl_kernel(expr, inputs) - self._guard_utf8_kernel_deps(col_deps) - return {"kind": "dsl", "kernel": kernel, "col_deps": col_deps} + if isinstance(expr, blosc2.DSLKernel | blosc2.PortableKernel): + names = expr.input_names if isinstance(expr, blosc2.DSLKernel) else list(expr.input_dtypes) + if inputs is None: + raise TypeError("A table kernel requires inputs= naming one source column per parameter") + if isinstance(inputs, Mapping): + bindings = dict(inputs) + else: + if len(inputs) != len(names): + raise ValueError(f"Kernel expects {len(names)} inputs, but received {len(inputs)}") + bindings = dict(zip(names, inputs, strict=True)) + self._guard_utf8_kernel_deps(bindings.values()) + if isinstance(expr, blosc2.DSLKernel): + output_dtype = dtype if isinstance(dtype, type) else getattr(dtype, "dtype", dtype) + dtype = self._dsl_result_dtype(expr, list(bindings.values()), output_dtype) + return self._portable_table_descriptor(expr, bindings, dtype=dtype, cardinality=cardinality) # Resolve a callable once (a lambda may return a LazyExpr or a LazyUDF). obj = expr(self._cols) if (callable(expr) and not isinstance(expr, blosc2.LazyExpr)) else expr if isinstance(obj, blosc2.LazyUDF): - if not isinstance(self._storage, InMemoryTableStorage): - raise TypeError("Persisting full DSL table columns is unsupported; use portable artifacts") if not isinstance(obj.func, blosc2.DSLKernel): raise TypeError( "Only LazyUDFs backed by a @blosc2.dsl_kernel are supported as CTable columns." @@ -13042,12 +13031,12 @@ def _normalize_transformer(self, expr, inputs=None) -> dict: raise blosc2.DSLSyntaxError(f"Invalid DSL kernel: {kernel.dsl_error}") col_deps = self._dsl_deps_from_lazyudf(obj) self._guard_utf8_kernel_deps(col_deps) - return { - "kind": "dsl", - "kernel": kernel, - "col_deps": col_deps, - "jit_backend": obj.kwargs.get("jit_backend"), - } + return self._normalize_transformer( + kernel, + dict(zip(kernel.input_names, col_deps, strict=True)), + dtype=obj.dtype if dtype is None else dtype, + cardinality=cardinality, + ) lazy, col_deps = self._normalize_expression_transformer(obj) # Guard: verify the expression string round-trips before storing. # An empty string means the LazyExpr was not fully constructed, and a @@ -13175,12 +13164,8 @@ def _generated_dependency_closure(self, source: str) -> set[str]: queue.append(name) return affected - def _portable_table_descriptor( - self, kernel, bindings, row_domain, *, dtype=None, cardinality="elementwise" - ): + def _portable_table_descriptor(self, kernel, bindings, *, dtype=None, cardinality="elementwise"): if isinstance(kernel, blosc2.DSLKernel): - if row_domain != "independent": - raise ValueError("Authored table kernels require explicit row_domain='independent'") if dtype is None: raise ValueError("Authored portable table kernels require an explicit output dtype") bindings = dict(bindings) @@ -13203,8 +13188,9 @@ def _portable_table_descriptor( ) if not isinstance(kernel, blosc2.PortableKernel): raise TypeError("Portable table transformers require a validated PortableKernel") - if row_domain != "independent": - raise ValueError("Only explicit independent-row portable table groups are implemented") + output_dtype = dtype if isinstance(dtype, type) else getattr(dtype, "dtype", dtype) + if dtype is not None and np.dtype(output_dtype) != kernel.output_dtype: + raise ValueError("Output dtype disagrees with the portable kernel signature") if kernel.schema_version != "1.0": raise ValueError("Independent rows require portable 1.0") bindings = dict(bindings) @@ -13227,7 +13213,6 @@ def _portable_table_descriptor( "kernel": kernel, "artifact": kernel.to_json(), "bindings": bindings, - "row_domain": row_domain, "row_shape": row_shape, "col_deps": list(dict.fromkeys(bindings.values())), "dtype": kernel.output_dtype, @@ -13235,7 +13220,9 @@ def _portable_table_descriptor( def _validate_portable_table_metadata(self, meta): kernel = blosc2.PortableKernel.from_json(meta["artifact"]) - desc = self._portable_table_descriptor(kernel, meta["bindings"], meta["row_domain"]) + if meta.get("row_domain", "independent") != "independent": + raise ValueError("Only independent-row portable table groups are implemented") + desc = self._portable_table_descriptor(kernel, meta["bindings"]) if ( desc["dtype"] != np.dtype(meta["dtype"]) or desc["col_deps"] != meta["col_deps"] @@ -13274,49 +13261,6 @@ def _evaluate_portable_rows(self, meta, raw_columns, *, count=None): result[row] = value[()] if value.shape == () else value[0] return result - def add_portable_computed_column( - self, name, kernel, *, inputs, row_domain, dtype=None, cardinality="elementwise" - ): - """Bind a native row transformer with explicit independent grouping. - - Each row's ND domain is (1,), origin (0,); append, refresh, deletion and - compaction do not change group membership or synthesize global row indices. - Fixed-shape row operands use their complete row domain for scalar native - reductions. Multi-row partitions/dynamic global coordinates reject explicitly. - A newly authored DSLKernel is normalized/validated before mutation; supply - dtype and, for row reductions, cardinality='block_scalar'. A PortableKernel - already carries that signature. The older vector-column DSL API does not - declare independent row grouping and is not converted implicitly. - """ - if self.base is not None or self._read_only: - raise ValueError("Portable columns require a writable root table") - _validate_column_name(name) - if name in self._cols or name in self._computed_cols: - raise ValueError("Column already exists") - desc = self._portable_table_descriptor( - kernel, inputs, row_domain, dtype=dtype, cardinality=cardinality - ) - self._computed_cols[name] = desc - self.col_names.append(name) - self._col_widths[name] = max(len(name), 15) - if isinstance(self._storage, FileTableStorage): - self._storage.save_schema(self._schema_dict_with_computed()) - - def add_portable_generated_column( - self, name, kernel, *, inputs, row_domain, dtype=None, cardinality="elementwise" - ): - """Store an independent-row native transformer, retaining it for append/refresh.""" - desc = self._portable_table_descriptor( - kernel, inputs, row_domain, dtype=dtype, cardinality=cardinality - ) - raw = {dep: self[dep][:] for dep in desc["col_deps"]} - values = self._evaluate_portable_rows(desc, raw, count=len(self)) - spec = self._coerce_generated_spec(None, values) - self.add_column(name, spec, values=values) - self._materialized_cols[name] = {**desc, "transformer_kind": "portable", "stale": False} - if isinstance(self._storage, FileTableStorage): - self._storage.save_schema(self._schema_dict_with_computed()) - def _mark_generated_columns_stale(self, source: str) -> None: affected = self._generated_dependency_closure(source) changed = False @@ -13450,11 +13394,13 @@ def add_generated_column( # noqa: C901 values: str | blosc2.LazyExpr | blosc2.DSLKernel + | blosc2.PortableKernel | Callable[[dict[str, Any]], blosc2.LazyExpr] | RowTransformer, dtype=None, create_index: bool = False, - inputs: list[str] | None = None, + inputs: Mapping[str, str] | list[str] | None = None, + cardinality: str = "elementwise", ) -> None: """Add a stored generated column maintained by the table. @@ -13470,7 +13416,8 @@ def add_generated_column( # noqa: C901 add_generated_column(name, *, values="price * qty", dtype=..., create_index=False) add_generated_column(name, *, values=lazy_expr, dtype=...) - add_generated_column(name, *, values=dsl_kernel, inputs=["price", "qty"], dtype=...) + add_generated_column(name, *, values=dsl_kernel, + inputs={"price": "price", "qty": "qty"}, dtype=...) add_generated_column(name, *, values=blosc2.lazyudf(dsl_kernel, (t.price, t.qty))) add_generated_column(name, *, values=lambda cols: cols["price"] * 1.21, dtype=...) add_generated_column(name, *, values=t.embedding.row_transformer.norm(axis=0), dtype=...) @@ -13491,17 +13438,18 @@ def add_generated_column( # noqa: C901 per row. * :class:`blosc2.LazyExpr`: scalar lazy expression over stored columns of this table. It must produce a 1-D scalar stream. - * :func:`blosc2.dsl_kernel`-decorated kernel passed directly with - ``inputs=[...]`` — one stored scalar column name per kernel - parameter, bound positionally. Produces one scalar per row. - The kernel source is persisted and recompiled on open; appended - rows are auto-filled and :meth:`refresh_generated_column` - recomputes after in-place edits. + * :class:`blosc2.DSLKernel` or :class:`blosc2.PortableKernel` with + named ``inputs={parameter: column}`` bindings (positional lists + are also accepted). Menudet artifacts are validated before + registration and persisted without Python reconstruction. + Each row is an independent logical group, including its complete + fixed-shape array inputs. Appended rows are auto-filled and + :meth:`refresh_generated_column` recomputes after in-place edits. * :class:`blosc2.LazyUDF` built from a :func:`blosc2.dsl_kernel` via :func:`blosc2.lazyudf` — column bindings are inferred by identity from the operands, so ``inputs=`` is not needed. Accepts :class:`Column` accessors (``t.col1``) or raw NDArrays as - operands. Same persistence and auto-fill behaviour as above. + operands. Same independent-row semantics and persistence as above. * callable: called as ``values(self._cols)`` and must return a :class:`blosc2.LazyExpr` or a :class:`blosc2.LazyUDF` backed by a :func:`blosc2.dsl_kernel`. @@ -13512,9 +13460,10 @@ def add_generated_column( # noqa: C901 may produce either one scalar per row or one fixed-shape ndarray item per row. - Expression and DSL forms currently cannot depend on computed columns - and cannot directly consume fixed-shape ndarray columns; use a - row-transformer for ndarray row projections/reductions. + These forms cannot depend on computed columns. Expressions cannot + consume fixed-shape ndarray columns; kernels support scalar row + reductions with ``cardinality="block_scalar"``. Row transformers + also support fixed-shape vector outputs. dtype: Output schema or dtype. Scalar outputs may pass a NumPy dtype or a Blosc2 scalar spec such as ``blosc2.float64()``. Fixed-shape @@ -13529,10 +13478,13 @@ def add_generated_column( # noqa: C901 generated columns raise :class:`ValueError` when indexing is requested. inputs: - Only used when *values* is a bare :func:`blosc2.dsl_kernel`: a list - of stored scalar column names, one per kernel parameter, bound - positionally. Not needed when passing a :class:`blosc2.LazyUDF` or - a callable — bindings are inferred from the operands in those cases. + For bare kernels, a mapping from parameter names to stored columns, + or a positional list of column names. LazyUDF bindings are inferred + from their operands. + cardinality: + Result contract for authored kernels: ``"elementwise"`` by default; + use ``"block_scalar"`` for fixed-shape row reductions. A + :class:`blosc2.PortableKernel` already carries its result contract. Examples -------- @@ -13639,36 +13591,19 @@ def add_generated_column( # noqa: C901 "transformer": transformer.to_metadata(), "stale": False, } - elif (desc := self._normalize_transformer(values, inputs))["kind"] == "dsl": - kernel = desc["kernel"] - col_deps = desc["col_deps"] - compute_dtype = ( - np.dtype(getattr(dtype, "dtype", dtype)) - if dtype is not None - else self._dsl_result_dtype(kernel, col_deps, None) - ) - jit_backend = desc.get("jit_backend") - operands = tuple(self._cols[d] for d in col_deps) - generated_values = np.asarray( - blosc2.lazyudf(kernel, operands, dtype=compute_dtype, jit_backend=jit_backend).compute()[:] - ) - if generated_values.ndim != 1: - raise TypeError("DSL generated columns must produce a 1-D scalar result.") - generated_values = ( - generated_values[self._valid_rows[:]] - if len(generated_values) == len(self._valid_rows) - else generated_values - ) + elif (desc := self._normalize_transformer(values, inputs, dtype=dtype, cardinality=cardinality))[ + "kind" + ] == "portable": + raw = {dep: self[dep][:] for dep in desc["col_deps"]} + generated_values = self._evaluate_portable_rows(desc, raw, count=n_live) spec = self._coerce_generated_spec(dtype, generated_values) metadata = { + **desc, "computed_column": None, "expression": None, - "dsl_source": kernel.dsl_source, - "col_deps": col_deps, "dtype": np.dtype(spec.dtype), - "transformer_kind": "dsl", + "transformer_kind": "portable", "stale": False, - "jit_backend": jit_backend, } else: lazy, col_deps = desc["lazy"], desc["col_deps"] @@ -13718,10 +13653,15 @@ def add_generated_column( # noqa: C901 def add_computed_column( self, name: str, - expr: str | blosc2.LazyExpr | blosc2.DSLKernel | Callable[[dict[str, Any]], blosc2.LazyExpr], + expr: str + | blosc2.LazyExpr + | blosc2.DSLKernel + | blosc2.PortableKernel + | Callable[[dict[str, Any]], blosc2.LazyExpr], *, dtype: np.dtype | None = None, - inputs: list[str] | None = None, + inputs: Mapping[str, str] | list[str] | None = None, + cardinality: str = "elementwise", ) -> None: """Add a read-only virtual column computed from stored columns. @@ -13737,7 +13677,7 @@ def add_computed_column( add_computed_column(name, "price * qty") add_computed_column(name, lazy_expr) - add_computed_column(name, dsl_kernel, inputs=["price", "qty"]) + add_computed_column(name, dsl_kernel, inputs={"price": "price", "qty": "qty"}) add_computed_column(name, blosc2.lazyudf(dsl_kernel, (t.price, t.qty))) add_computed_column(name, lambda cols: cols["price"] * cols["qty"]) @@ -13753,11 +13693,10 @@ def add_computed_column( ``"price * qty"``. * :class:`blosc2.LazyExpr`: lazy expression over stored columns of this table. - * :func:`blosc2.dsl_kernel`-decorated kernel passed directly with - ``inputs=[...]`` — one stored scalar column name per kernel - parameter, bound positionally. The kernel may use loops, - ``if``/``else`` and ``where(...)``. Its source is persisted and - recompiled on open; the column stays virtual/unstored. + * :class:`blosc2.DSLKernel` or :class:`blosc2.PortableKernel` with + named ``inputs={parameter: column}`` bindings (positional lists + are also accepted). Kernels are validated as Menudet before + registration and persist without Python reconstruction. * :class:`blosc2.LazyUDF` built from a :func:`blosc2.dsl_kernel` via :func:`blosc2.lazyudf` — column bindings are inferred by identity from the operands, so ``inputs=`` is not needed. Accepted forms @@ -13767,16 +13706,19 @@ def add_computed_column( :class:`blosc2.LazyExpr` or a :class:`blosc2.LazyUDF` backed by a :func:`blosc2.dsl_kernel`. - DSL columns (last three forms) are persisted — their source is stored - and recompiled on open — and may be referenced inside :meth:`where` - predicates. + Kernel columns use independent-row semantics: scalar inputs form + one-lane groups and fixed-shape inputs supply their complete row. + Reductions never combine different rows. The column remains virtual + and may be referenced inside :meth:`where` predicates. Loaded legacy + table recipes require ``deserialize="full"`` and retain their old + block/batch-dependent semantics, rather than being reinterpreted. Expressions must depend only on stored columns of this table; computed columns cannot depend on other computed columns in this version. Fixed-shape ndarray columns are not accepted in computed - column expressions yet. For row-wise ndarray projections or - reductions, use :meth:`add_generated_column` with - ``values=t.ndarray_col.row_transformer...``. + column expressions yet. Kernels can reduce complete ndarray rows + with ``cardinality="block_scalar"``; generated row transformers + additionally support vector-valued outputs. dtype: Optional dtype override for the computed values. For expression forms it is inferred from the resulting :class:`blosc2.LazyExpr` @@ -13787,11 +13729,13 @@ def add_computed_column( column inputs. This changes the dtype reported by the CTable column wrapper; it does not create physical storage. inputs: - Only used when *expr* is a bare :func:`blosc2.dsl_kernel`: a list of - stored scalar column names, one per kernel parameter, bound - positionally (kernel parameter ``i`` ← ``inputs[i]``). Not needed - when passing a :class:`blosc2.LazyUDF` or a callable — bindings are - inferred from the operands in those cases. + For bare kernels, a mapping from parameter names to stored columns, + or a positional list of column names. LazyUDF bindings are inferred + from their operands. + cardinality: + Result contract for authored kernels: ``"elementwise"`` by default; + use ``"block_scalar"`` for fixed-shape row reductions. A + :class:`blosc2.PortableKernel` already carries its result contract. Examples -------- @@ -13869,18 +13813,9 @@ def ratio(num, den): if name in self._computed_cols: raise ValueError(f"A computed column named {name!r} already exists.") - desc = self._normalize_transformer(expr, inputs) - if desc["kind"] == "dsl": - kernel = desc["kernel"] - col_deps = desc["col_deps"] - self._computed_cols[name] = { - "kind": "dsl", - "dsl_source": kernel.dsl_source, - "kernel": kernel, - "col_deps": col_deps, - "dtype": self._dsl_result_dtype(kernel, col_deps, dtype), - "jit_backend": desc.get("jit_backend"), - } + desc = self._normalize_transformer(expr, inputs, dtype=dtype, cardinality=cardinality) + if desc["kind"] == "portable": + self._computed_cols[name] = desc else: lazy = desc["lazy"] self._computed_cols[name] = { @@ -14015,17 +13950,8 @@ def assign(self, **named_exprs) -> CTable: view._col_widths = dict(self._col_widths) for name, value in bound.items(): desc = view._normalize_transformer(value) - if desc["kind"] == "dsl": - kernel = desc["kernel"] - col_deps = desc["col_deps"] - view._computed_cols[name] = { - "kind": "dsl", - "dsl_source": kernel.dsl_source, - "kernel": kernel, - "col_deps": col_deps, - "dtype": view._dsl_result_dtype(kernel, col_deps, None), - "jit_backend": desc.get("jit_backend"), - } + if desc["kind"] == "portable": + view._computed_cols[name] = desc else: lazy = desc["lazy"] view._computed_cols[name] = { diff --git a/src/blosc2/exceptions.py b/src/blosc2/exceptions.py index 8faae5d52..7854a8a4b 100644 --- a/src/blosc2/exceptions.py +++ b/src/blosc2/exceptions.py @@ -18,12 +18,14 @@ def __init__(self, expr, missing_ops): class UnsafeDeserializationError(ValueError): """Raised when safe deserialization encounters an active serialized value.""" - def __init__(self, kind: str, *, location: str | None = None): + def __init__(self, kind: str, *, location: str | None = None, hint: str | None = None): where = f" in {location}" if location else "" - super().__init__( - f"Encountered active serialized value {kind!r}{where} while using " - "deserialize='safe'. Reopen with deserialize='full' only if you trust " + advice = hint or ( + "Reopen with deserialize='full' only if you trust " "this data and intend to allow reference resolution and lazy-object reconstruction." ) + super().__init__( + f"Encountered active serialized value {kind!r}{where} while using deserialize='safe'. {advice}" + ) self.kind = kind self.location = location diff --git a/tests/ctable/test_ctable_computed_cols.py b/tests/ctable/test_ctable_computed_cols.py index f50e22d93..56a13ab47 100644 --- a/tests/ctable/test_ctable_computed_cols.py +++ b/tests/ctable/test_ctable_computed_cols.py @@ -878,9 +878,7 @@ def test_dsl_computed_column_no_storage(): def test_dsl_computed_column_persistence(tmp_path): """Authored computed columns persist with an explicit row-domain contract.""" t = _make_invoice_table(5) - t.add_portable_computed_column( - "total", total, inputs={"price": "price", "qty": "qty"}, dtype="float64", row_domain="independent" - ) + t.add_computed_column("total", total, inputs={"price": "price", "qty": "qty"}, dtype="float64") path = str(tmp_path / "tbl") t.save(path) restored = blosc2.CTable.open(path) @@ -983,9 +981,7 @@ def test_dsl_generated_column_autofill_append(): def test_dsl_generated_column_persistence_and_append(tmp_path): """Authored row transformers persist and refill appended rows natively.""" t = _make_invoice_table(3) - t.add_portable_generated_column( - "total", total, inputs={"price": "price", "qty": "qty"}, dtype="float64", row_domain="independent" - ) + t.add_generated_column("total", values=total, inputs={"price": "price", "qty": "qty"}, dtype="float64") path = str(tmp_path / "tbl") t.save(path) restored = blosc2.CTable.open(path, mode="a") @@ -1009,8 +1005,8 @@ def test_dsl_generated_column_refresh(tmp_path): @pytest.mark.skipif(sys.platform == "win32", reason="cc backend requires a system C compiler") -def test_dsl_jit_backend_persisted_in_schema(): - """jit_backend set via lazyudf() is stored in the column schema.""" +def test_dsl_jit_backend_not_persisted_in_portable_schema(): + """A local JIT preference does not become a portable execution requirement.""" t = _make_invoice_table(3) t.add_generated_column( "total", @@ -1018,20 +1014,25 @@ def test_dsl_jit_backend_persisted_in_schema(): ) schema = t._schema_dict_with_computed() mat = {m["name"]: m for m in schema["materialized_columns"]} - assert mat["total"]["jit_backend"] == "cc" + assert mat["total"]["transformer_kind"] == "portable" + assert "jit_backend" not in mat["total"] + assert "dsl_source" not in mat["total"] @pytest.mark.skipif(sys.platform == "win32", reason="cc backend requires a system C compiler") -def test_dsl_jit_backend_restored_after_open(tmp_path): - """Selecting a JIT backend cannot bypass portable persistence preflight.""" +def test_dsl_jit_backend_falls_back_after_portable_open(tmp_path): + """A local backend preference allows native portable persistence/fallback.""" t = _make_invoice_table(3) t.add_generated_column( "total", values=blosc2.lazyudf(total, (t.price, t.qty), jit_backend="cc"), ) path = str(tmp_path / "tbl") - with pytest.raises(TypeError, match="row-domain"): - t.save(path) + t.save(path) + restored = blosc2.CTable.open(path, mode="a") + restored.append({"price": 4.0, "qty": 4}) + np.testing.assert_allclose(restored["total"][:], [1.0, 4.0, 9.0, 16.0]) + restored.close() def test_dsl_no_jit_backend_not_in_schema(): diff --git a/tests/ctable/test_ctable_dsl_columns.py b/tests/ctable/test_ctable_dsl_columns.py index 5f514c2b5..cc40bb363 100644 --- a/tests/ctable/test_ctable_dsl_columns.py +++ b/tests/ctable/test_ctable_dsl_columns.py @@ -86,6 +86,13 @@ def test_dsl_computed_callable_lazyudf_form(): np.testing.assert_array_equal(np.asarray(t["r"][:]), np.arange(20) + 20) +def test_assign_dsl_lazyudf_uses_portable_rows(): + t = _make_table() + view = t.assign(r=blosc2.lazyudf(k_loop, (t.a, t.b), dtype=np.int64)) + assert view._computed_cols["r"]["kind"] == "portable" + np.testing.assert_array_equal(view["r"][:], np.arange(20) + 20) + + def test_dsl_computed_partial_slice(): t = _make_table() t.add_computed_column("r", k_add, inputs=["a", "b"]) @@ -152,17 +159,17 @@ def test_dsl_computed_in_where_streams_multichunk(): def test_dsl_computed_roundtrip(tmp_path): path = str(tmp_path / "dsl.b2d") t = _make_table(urlpath=path, mode="w") - with pytest.raises(TypeError, match="portable artifacts"): - t.add_computed_column("r", k_loop, inputs=["a", "b"]) + t.add_computed_column("r", k_loop, inputs={"x": "a", "y": "b"}) t.close() - # Rejection leaves no source-backed computed recipe in the stored schema. + # New registration stores an artifact rather than Python source. meta = blosc2.open(f"{path}/_meta.b2f") sd = json.loads(meta.vlmeta["schema"]) - assert "computed_columns" not in sd + assert sd["computed_columns"][0]["kind"] == "portable" + assert "dsl_source" not in sd["computed_columns"][0] t2 = blosc2.open(path) - assert "r" not in t2.col_names + np.testing.assert_array_equal(t2["r"][:], np.arange(20) + 20) t2.close() @@ -224,14 +231,14 @@ def test_dsl_generated_create_index(): def test_dsl_generated_roundtrip(tmp_path): path = str(tmp_path / "gen.b2d") t = _make_table(n=10, urlpath=path, mode="w") - with pytest.raises(TypeError, match="portable artifacts"): - t.add_generated_column("g", values=k_add, inputs=["a", "b"], dtype=blosc2.int64()) + t.add_generated_column("g", values=k_add, inputs={"x": "a", "y": "b"}, dtype=blosc2.int64()) t.close() meta = blosc2.open(f"{path}/_meta.b2f") sd = json.loads(meta.vlmeta["schema"]) - assert "materialized_columns" not in sd + assert sd["materialized_columns"][0]["transformer_kind"] == "portable" + assert "dsl_source" not in sd["materialized_columns"][0] t2 = blosc2.open(path) - assert "g" not in t2.col_names + np.testing.assert_array_equal(t2["g"][:], np.arange(10) + 10) t2.close() diff --git a/tests/ndarray/test_lazyexpr.py b/tests/ndarray/test_lazyexpr.py index 5a099cbac..bb74a1eb7 100644 --- a/tests/ndarray/test_lazyexpr.py +++ b/tests/ndarray/test_lazyexpr.py @@ -487,7 +487,7 @@ def test_arctan2_pow(urlpath, shape_fixture, dtype_fixture, function, value1, va expr = blosc2.LazyExpr(new_op=(a1, function, a2)) if urlpath is not None: expr.save(urlpath=urlpath_save) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr if function == "**": @@ -501,7 +501,7 @@ def test_arctan2_pow(urlpath, shape_fixture, dtype_fixture, function, value1, va expr = blosc2.LazyExpr(new_op=(a1, function, value2)) if urlpath is not None: expr.save(urlpath=urlpath_save) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr if function == "**": @@ -517,7 +517,7 @@ def test_arctan2_pow(urlpath, shape_fixture, dtype_fixture, function, value1, va expr = blosc2.LazyExpr(new_op=(value1, function, a2)) if urlpath is not None: expr.save(urlpath=urlpath_save) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr if function == "**": @@ -737,7 +737,7 @@ def test_save(): ) np.testing.assert_allclose(res[:], nres, rtol=tol, atol=tol) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") # After opening, check that a lazy expression does have an array # and schunk attributes. This is to allow the .info() method to work. assert hasattr(expr, "array") is True @@ -756,7 +756,7 @@ def test_save(): var_dict = {"a1": ops[0], "a2": ops[1], "a3": ops[2], "a4": ops[3], "x": x} lazy_expr = eval(expr, var_dict) lazy_expr.save(urlpath=urlpath_save2) - expr = blosc2.open(urlpath_save2, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save2, mode="r") assert expr.array.dtype == np.float64 res = expr.compute() nres = ne_evaluate("na1 / na2 + na2 - na3 * na4**3") @@ -780,7 +780,7 @@ def test_save_unsafe(): expr.save(urlpath=urlpath) disk_arrays.append(urlpath) - expr = blosc2.open(urlpath, mode="r", deserialize="full") + expr = blosc2.open(urlpath, mode="r") # Replace expression by a (potentially) unsafe expression expr.expression = "import os; os.system('touch /tmp/unsafe')" with pytest.raises(ValueError) as excinfo: @@ -828,7 +828,7 @@ def test_save_functions(function, dtype_fixture, shape_fixture): expr = blosc2.LazyExpr(new_op=(a1, function, None)) expr.save(urlpath=urlpath_save) del expr - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() # Evaluate using NumExpr @@ -844,7 +844,7 @@ def test_save_functions(function, dtype_fixture, shape_fixture): res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol) @@ -868,7 +868,7 @@ def test_save_contains(values): # Construct the lazy expression expr_lazy = blosc2.LazyExpr(new_op=(a1_blosc, "contains", value2)) expr_lazy.save(urlpath=urlpath_save) - expr_lazy = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr_lazy = blosc2.open(urlpath_save, mode="r") # Evaluate using NumExpr expr_numexpr = f"{'contains'}(a1, value2)" res_numexpr = ne_evaluate(expr_numexpr) @@ -878,7 +878,7 @@ def test_save_contains(values): # Construct the lazy expression expr_lazy = blosc2.LazyExpr(new_op=(a1_blosc, "contains", a2_blosc)) expr_lazy.save(urlpath=urlpath_save) - expr_lazy = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr_lazy = blosc2.open(urlpath_save, mode="r") # Evaluate using NumExpr res_numexpr = ne_evaluate("contains(a2, a1)") else: # ("str", "NDArray") @@ -888,7 +888,7 @@ def test_save_contains(values): # Construct the lazy expression expr_lazy = blosc2.LazyExpr(new_op=(value1, "contains", a2_blosc)) expr_lazy.save(urlpath=urlpath_save) - expr_lazy = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr_lazy = blosc2.open(urlpath_save, mode="r") # Evaluate using NumExpr res_numexpr = ne_evaluate("contains(value1, a2)") res_lazyexpr = expr_lazy.compute() @@ -922,7 +922,7 @@ def test_save_many_functions(dtype_fixture, shape_fixture): res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol, atol=atol) - expr = blosc2.open(urlpath_save, mode="r", deserialize="full") + expr = blosc2.open(urlpath_save, mode="r") res_lazyexpr = expr.compute() np.testing.assert_allclose(res_lazyexpr[:], res_numexpr, rtol=rtol, atol=atol) @@ -976,7 +976,7 @@ def test_save_constructor(disk, shape, dtype, constructor): a = b2func(lshape, dtype=dtype, shape=shape, urlpath=urlpath, mode="w") expr = f"a + {constructor}({lshape}, dtype={dtype}, shape={shape}) + 1" if disk: - a = blosc2.open(urlpath, mode="r", deserialize="full") + a = blosc2.open(urlpath, mode="r") npfunc = getattr(np, constructor) if constructor == "linspace": na = npfunc(0, 10, lshape, dtype=dtype).reshape(shape) @@ -994,7 +994,7 @@ def test_save_constructor(disk, shape, dtype, constructor): assert lexpr.shape == a.shape if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() nres = na + na + 1 assert np.allclose(res[()], nres) @@ -1016,7 +1016,7 @@ def test_save_2_constructors(shape, disk): lexpr = blosc2.lazyexpr(expr) if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() na = np.arange(lshape).reshape(shape) nb = np.ones(shape) @@ -1043,7 +1043,7 @@ def test_save_constructor_reshape(shape, disk): lexpr = blosc2.lazyexpr(expr) if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() na = np.arange(lshape).reshape(shape) nb = np.ones(shape) @@ -1067,7 +1067,7 @@ def test_save_2equal_constructors(shape, disk): lexpr = blosc2.lazyexpr(expr) if disk: lexpr.save("out.b2nd") - lexpr = blosc2.open("out.b2nd", mode="r", deserialize="full") + lexpr = blosc2.open("out.b2nd", mode="r") res = lexpr.compute() na = np.ones(shape, dtype=np.int8) nb = np.ones(shape) @@ -1374,9 +1374,9 @@ def test_fill_disk_operands(chunks, blocks, disk, fill_value): b = blosc2.zeros((N, N), urlpath=bpath, mode="w", chunks=chunks, blocks=blocks) c = blosc2.zeros((N, N), urlpath=cpath, mode="w", chunks=chunks, blocks=blocks) if disk: - a = blosc2.open("a.b2nd", mode="r", deserialize="full") - b = blosc2.open("b.b2nd", mode="r", deserialize="full") - c = blosc2.open("c.b2nd", mode="r", deserialize="full") + a = blosc2.open("a.b2nd", mode="r") + b = blosc2.open("b.b2nd", mode="r") + c = blosc2.open("c.b2nd", mode="r") expr = ((a**3 + blosc2.sin(c * 2)) < b) & ~(c > 0) @@ -1720,7 +1720,7 @@ def test_missing_operator(): blosc2.remove_urlpath("b.b2nd") # Re-open the lazy expression with pytest.raises(blosc2.exceptions.MissingOperands) as excinfo: - blosc2.open("expr.b2nd", mode="r", deserialize="full") + blosc2.open("expr.b2nd", mode="r") # Check that some operand is missing assert "a" not in excinfo.value.missing_ops @@ -1751,13 +1751,13 @@ def test_save_dictstore_operands(tmp_path): expr = blosc2.lazyexpr("a + b") expr.save(expr_path) - carrier = blosc2.open(expr_path, mode="r", deserialize="full").array + carrier = blosc2.open(expr_path, mode="r").array assert carrier.schunk.vlmeta["b2o"]["operands"] == { "a": {"kind": "dictstore_key", "version": 1, "urlpath": str(store_path), "key": "/a"}, "b": {"kind": "dictstore_key", "version": 1, "urlpath": str(store_path), "key": "/b"}, } - restored = blosc2.open(expr_path, mode="r", deserialize="full") + restored = blosc2.open(expr_path, mode="r") assert isinstance(restored, blosc2.LazyExpr) np.testing.assert_array_equal(restored[:], expected) @@ -1773,6 +1773,9 @@ def test_save_proxy_operands_reopen_default_mode(tmp_path): expr = proxy + proxy expr.save(str(expr_path)) + # Proxy reconstruction remains full-only, even inside an ordinary expression. + with pytest.raises(blosc2.UnsafeDeserializationError, match="proxy"): + blosc2.open(str(expr_path), mode="r") restored = blosc2.open(str(expr_path), mode="r", deserialize="full") assert isinstance(restored, blosc2.LazyExpr) @@ -1795,7 +1798,7 @@ def test_lazyexpr_vlmeta_in_memory_and_persisted(tmp_path): expr_path = tmp_path / "expr_vlmeta.b2nd" expr.save(str(expr_path)) - restored = blosc2.open(str(expr_path), mode="r", deserialize="full") + restored = blosc2.open(str(expr_path), mode="r") assert restored.vlmeta["name"] == "sum" assert restored.vlmeta["config"] == {"scale": 1} @@ -1803,9 +1806,9 @@ def test_lazyexpr_vlmeta_in_memory_and_persisted(tmp_path): with pytest.raises(ValueError, match="reading mode"): restored.vlmeta["note"] = "persisted" - writable = blosc2.open(str(expr_path), mode="a", deserialize="full") + writable = blosc2.open(str(expr_path), mode="a") writable.vlmeta["note"] = "persisted" - reopened = blosc2.open(str(expr_path), mode="r", deserialize="full") + reopened = blosc2.open(str(expr_path), mode="r") assert reopened.vlmeta["note"] == "persisted" np.testing.assert_array_equal(reopened[:], np.arange(5, dtype=np.int64) * 2) @@ -1881,19 +1884,19 @@ def test_chain_persistentexpressions(): le1_ = blosc2.lazyexpr("a ** 3 + sin(a ** 2)", {"a": a}) le1_.save("expr1.b2nd", mode="w") - myle1 = blosc2.open("expr1.b2nd", mode="r", deserialize="full") + myle1 = blosc2.open("expr1.b2nd", mode="r") le2_ = blosc2.lazyexpr("(le1 < c)", {"le1": myle1, "c": c}) le2_.save("expr2.b2nd", mode="w") - myle2 = blosc2.open("expr2.b2nd", mode="r", deserialize="full") + myle2 = blosc2.open("expr2.b2nd", mode="r") le3_ = blosc2.lazyexpr("(b < 0)", {"b": b}) le3_.save("expr3.b2nd", mode="w") - myle3 = blosc2.open("expr3.b2nd", mode="r", deserialize="full") + myle3 = blosc2.open("expr3.b2nd", mode="r") le4_ = blosc2.lazyexpr("(le2 & le3)", {"le2": myle2, "le3": myle3}) le4_.save("expr4.b2nd", mode="w") - myle4 = blosc2.open("expr4.b2nd", mode="r", deserialize="full") + myle4 = blosc2.open("expr4.b2nd", mode="r") assert (myle4[:] == le4[:]).all() # Remove files diff --git a/tests/test_deserialization.py b/tests/test_deserialization.py index 38b9a5d56..8e3cba9c8 100644 --- a/tests/test_deserialization.py +++ b/tests/test_deserialization.py @@ -186,6 +186,17 @@ def test_active_top_level_object_requires_full(tmp_path): np.testing.assert_array_equal(trusted[:], np.arange(4) + 1) +@pytest.mark.parametrize("expression", ["sqrt(x)", "sum(x)", "x.mean()", "x[0]"]) +def test_ordinary_lazyexpr_roundtrip_defaults_safe(tmp_path, expression): + values = np.arange(1, 5, dtype="float64") + operand = blosc2.asarray(values, urlpath=tmp_path / "operand.b2nd", mode="w") + lazy = blosc2.lazyexpr(expression, operands={"x": operand}) + expected = lazy[()] + path = tmp_path / "expression.b2nd" + lazy.save(path) + np.testing.assert_allclose(blosc2.open(path)[()], expected) + + @pytest.mark.parametrize("value", [True, False, "unknown", None]) def test_invalid_deserialize_mode(value): source = blosc2.SChunk() diff --git a/tests/test_portable_policy.py b/tests/test_portable_policy.py index 1250e645e..8df12dba7 100644 --- a/tests/test_portable_policy.py +++ b/tests/test_portable_policy.py @@ -10,13 +10,14 @@ @pytest.mark.parametrize("policy", ["safe", "full"]) -def test_nested_legacy_effective_policy(tmp_path, policy): +@pytest.mark.parametrize("expression", ["x + 1", "sqrt(x)"]) +def test_nested_legacy_effective_policy(tmp_path, policy, expression): fixture = Path(__file__).parent / "data" / "legacy_lazyudf_v1" / "expr.b2nd" carrier = blosc2.empty((5,), dtype="float64", meta={"b2o": {"kind": "lazyexpr", "version": 1}}) carrier.schunk.vlmeta["b2o"] = { "kind": "lazyexpr", "version": 1, - "expression": "x + 1", + "expression": expression, "operands": {"x": {"kind": "urlpath", "version": 1, "urlpath": str(fixture)}}, } if policy == "safe": @@ -97,9 +98,58 @@ def test_ctable_legacy_metadata_policy_before_reconstruction(monkeypatch): {"name": "bad", "col_deps": [], "dtype": "int64", "kind": "dsl", "dsl_source": "invalid"} ] } - with pytest.raises(blosc2.UnsafeDeserializationError): + with pytest.raises( + blosc2.UnsafeDeserializationError, match=r"block/batch-dependent.*deserialize='full'" + ): table._load_computed_cols_from_schema(schema) table._computed_cols = {"x": {"kind": "dsl"}} table._materialized_cols = {} with pytest.raises(TypeError, match="row-domain"): table._preflight_portable_persistence() + + +@pytest.mark.parametrize("generated", [False, True]) +def test_legacy_table_recipes_retain_full_opt_in(tmp_path, generated): + from dataclasses import dataclass + + @dataclass + class Row: + x: float = 0.0 + + path = str(tmp_path / "legacy.b2d") + table = blosc2.CTable( + Row, urlpath=path, mode="w", new_data={"x": [2.0, 4.0]}, create_summary_index=False + ) + if generated: + table.add_column("old", blosc2.float64(), values=np.array([4.0, 6.0])) + schema = table._schema_dict_with_computed() + recipe = { + "name": "old", + "dsl_source": "def old(x):\n return x + sum(x)\n", + "col_deps": ["x"], + "dtype": "float64", + } + if generated: + recipe.update(transformer_kind="dsl", stale=False, computed_column=None, expression=None) + schema["materialized_columns"] = [recipe] + else: + recipe["kind"] = "dsl" + schema["computed_columns"] = [recipe] + table._storage.save_schema(schema) + table.close() + with pytest.raises( + blosc2.UnsafeDeserializationError, match=r"block/batch-dependent.*deserialize='full'" + ): + blosc2.CTable.open(path) + trusted = blosc2.CTable.open(path, mode="a", deserialize="full") + if generated: + expected_batch = trusted._evaluate_dsl_materialized_batch( + trusted._materialized_cols["old"], {"x": np.array([8.0])} + ) + trusted.append({"x": 8.0}) + # Appended batches retain their old groups rather than adopting the new API contract. + np.testing.assert_array_equal(trusted["old"][:], [4.0, 6.0, expected_batch[0]]) + else: + # The original two-row execution block, not independent one-lane rows. + np.testing.assert_array_equal(trusted["old"][:], [8.0, 10.0]) + trusted.close() diff --git a/tests/test_portable_table_export.py b/tests/test_portable_table_export.py index b0ff79033..cb15e1167 100644 --- a/tests/test_portable_table_export.py +++ b/tests/test_portable_table_export.py @@ -138,10 +138,12 @@ class Row: source = blosc2.DSLKernel.from_source("def rows(x):\n return sum(x) + _flat_idx + _n0\n") kernel = blosc2.PortableKernel.from_json(source.export({"x": "int64"}, "int64", version="1.0", ndim=1)) table = blosc2.CTable(Row, new_data={"x": [2, 4]}, create_summary_index=False) - table.add_portable_computed_column("virtual", kernel, inputs={"x": "x"}, row_domain="independent") - table.add_portable_generated_column( - "stored", source, dtype="int64", inputs={"x": "x"}, row_domain="independent" - ) + table.add_computed_column("virtual", kernel, inputs={"x": "x"}) + old_draft = {**table._computed_cols["virtual"], "row_domain": "independent"} + assert table._validate_portable_table_metadata(old_draft)["kind"] == "portable" + with pytest.raises(ValueError, match="independent"): + table._validate_portable_table_metadata({**old_draft, "row_domain": "dynamic"}) + table.add_generated_column("stored", values=source, dtype="int64", inputs={"x": "x"}) np.testing.assert_array_equal(table["virtual"][:], [3, 5]) table.append({"x": 8}) np.testing.assert_array_equal(table["stored"][:], [3, 5, 9]) @@ -173,7 +175,7 @@ class Row: np.testing.assert_array_equal(copied["stored"][:], [3, 9, 11, 13, 15]) constructor = blosc2.DSLKernel.from_source("def constant_row():\n return _n0\n") constructed = blosc2.PortableKernel.from_json(constructor.export({}, "int64", version="1.0", ndim=1)) - copied.add_portable_generated_column("constant", constructed, inputs={}, row_domain="independent") + copied.add_generated_column("constant", values=constructed, inputs={}) copied.append({"x": 16}) copied.refresh_generated_column("constant") np.testing.assert_array_equal(copied["constant"][:], np.ones(6, dtype="int64")) @@ -183,8 +185,10 @@ class Row: with pytest.raises(blosc2.PortableArtifactError): copied.save(str(sentinel)) assert sentinel.read_bytes() == b"original destination" - with pytest.raises(ValueError, match="independent"): - table.add_portable_computed_column("invalid", kernel, inputs={"x": "x"}, row_domain="dynamic") + with pytest.raises(TypeError, match="row_domain"): + table.add_computed_column("invalid", kernel, inputs={"x": "x"}, row_domain="dynamic") + assert not hasattr(table, "add_portable_computed_column") + assert not hasattr(table, "add_portable_generated_column") @dataclass class TextRow: @@ -192,9 +196,7 @@ class TextRow: text_table = blosc2.CTable(TextRow, new_data={"label": ["ß a", "ab"]}, create_summary_index=False) upper = blosc2.DSLKernel.from_source("def upper_row(x):\n return upper(x)\n") - text_table.add_portable_generated_column( - "upper", upper, dtype="U4", inputs={"x": "label"}, row_domain="independent" - ) + text_table.add_generated_column("upper", values=upper, dtype="U4", inputs={"x": "label"}) text_table.append({"label": "z"}) np.testing.assert_array_equal(text_table["upper"][:], ["SS A", "AB", "Z"]) @@ -207,15 +209,53 @@ class VectorRow: VectorRow, new_data={"vector": [[1, 2, 3], [4, 5, 6]], "bias": [1, 2]}, create_summary_index=False ) total = blosc2.DSLKernel.from_source("def total_row(x, bias):\n return sum(x + bias) + _n0\n") - vector_table.add_portable_generated_column( + vector_table.add_generated_column( "total", - total, + values=total, dtype="int64", cardinality="block_scalar", inputs={"x": "vector", "bias": "bias"}, - row_domain="independent", ) vector_table.append({"vector": [7, 8, 9], "bias": 3}) vector_table.refresh_generated_column("total") restored = blosc2.ctable_from_cframe(vector_table.to_cframe()) np.testing.assert_array_equal(restored["total"][:], [12, 24, 36]) + + +@pytest.mark.parametrize("generated", [False, True]) +def test_unified_table_registration_is_row_local_and_validated(tmp_path, generated): + @dataclass + class Row: + amount: float = 0.0 + + table = blosc2.CTable( + Row, + urlpath=str(tmp_path / "rows.b2d"), + mode="w", + new_data={"amount": [2.0, 4.0, 8.0]}, + create_summary_index=False, + ) + kernel = blosc2.DSLKernel.from_source("def center(x):\n return x - mean(x)\n") + if generated: + table.add_generated_column("centered", values=kernel, inputs={"x": "amount"}, create_index=True) + else: + table.add_computed_column("centered", kernel, inputs={"x": "amount"}) + np.testing.assert_array_equal(table["centered"][:], [0, 0, 0]) + table.append({"amount": 16.0}) + np.testing.assert_array_equal(table["centered"][1:3], [0, 0]) + schema = table._schema_dict_with_computed() + recipe = schema["materialized_columns" if generated else "computed_columns"][0] + assert "row_domain" not in recipe + assert "dsl_source" not in recipe + invalid = blosc2.DSLKernel.from_source("def invalid(x):\n return callback(x)\n") + if generated: + with pytest.raises(blosc2.PortableArtifactError): + table.add_generated_column("invalid", values=invalid, inputs={"x": "amount"}) + else: + with pytest.raises(blosc2.PortableArtifactError): + table.add_computed_column("invalid", invalid, inputs={"x": "amount"}) + assert "invalid" not in table.col_names + table.close() + restored = blosc2.CTable.open(str(tmp_path / "rows.b2d")) + np.testing.assert_array_equal(restored["centered"][:], [0, 0, 0, 0]) + restored.close()