diff --git a/benchmark/matched_cellprofiler_batch.py b/benchmark/matched_cellprofiler_batch.py index 017edca9bb..c089e8292b 100644 --- a/benchmark/matched_cellprofiler_batch.py +++ b/benchmark/matched_cellprofiler_batch.py @@ -181,14 +181,46 @@ def _require_compared_output_inventory( compared_file_report: RuntimeEquivalenceReport = RuntimeEquivalenceReport(()), ) -> None: """Reject unqualified output formats and missing scientific output files.""" + # Validate all explicit edges against their object rows, then compare the + # cross-output correlations before using redundant scalar table inventory. + from openhcs.core.equivalence.relationships import ( + ExportedRelationshipMeasurementSemantics, + ) + from openhcs.core.runtime_equivalence import RuntimeMeasurementSnapshot + + measurements = tuple( + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + for snapshot in (reference_snapshot, candidate_snapshot) + ) + correlations = tuple( + measurement.required_relationship_correlations() for measurement in measurements + ) + if correlations[0] != correlations[1]: + raise RuntimeError("Matched saved output relationship correlations differ.") + comparable_tables = tuple( + tuple( + table + for table in snapshot.tables + if table.participates_in_comparison + and not ExportedRelationshipMeasurementSemantics.supports_table(table) + ) + for snapshot in (reference_snapshot, candidate_snapshot) + ) counts = [] - for files, exports, snapshot, managed_files in ( - (reference_files, reference_exports, reference_snapshot, frozenset()), + for files, exports, snapshot, managed_files, tables in ( + ( + reference_files, + reference_exports, + reference_snapshot, + frozenset(), + comparable_tables[0], + ), ( candidate_files, candidate_exports, candidate_snapshot, candidate_managed_files, + comparable_tables[1], ), ): files = files - managed_files @@ -208,7 +240,7 @@ def _require_compared_output_inventory( - len(snapshot.tables) - len(exports.image_outputs) + len(snapshot.images) - + sum(table.participates_in_comparison for table in snapshot.tables) + + len(tables) ) if count < 1: raise RuntimeError("Matched batch has no compared scientific output files.") @@ -230,10 +262,9 @@ def _require_compared_output_inventory( ), len(table.rows), ) - for table in snapshot.tables - if table.participates_in_comparison + for table in tables ) - for snapshot in (reference_snapshot, candidate_snapshot) + for tables in comparable_tables ) if table_shapes[0] != table_shapes[1]: raise RuntimeError( diff --git a/benchmark/results/perf_schema_source_readiness_20261003/README.rst b/benchmark/results/perf_schema_source_readiness_20261003/README.rst new file mode 100644 index 0000000000..17b2e5305c --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/README.rst @@ -0,0 +1,25 @@ +Registered source readiness: ordinary Illumination ABBA +==================================================== + +``receipt.json`` contains public observations, the exact variant hashes and full +science-qualification links. The committed recipes preserve the actual local +experiment; they are evidence, not a turnkey portable benchmark command. Their +paths name the owned checkout and immutable maintenance directory, and the +ordinary helper/bootstrap and physical dataset/native references must already +exist. Do not rerun into a completed output directory. + +Only three source files vary. Their baseline bytes are derived from the recorded +Git revisions in ``variants.json``. Candidate registry bytes are available at +OpenHCS commit ``7258826d7198f2bb7010763276dba02320d62142``; candidate dependency +bytes are available at python-introspect merge +``6f5ac0b79d6cef163df6a8389e72a889a91ed548`` and ObjectState merge +``55e71df39a05aa08d00bc1bb06c2be174ad2c522``. The original maintenance directory +retains all six variant files, source freezes and four immutable output trees. +No dependency source, benchmark images or environments are duplicated here. + +Mean compilation is 0.962152 → 0.587605 seconds; total is +1.693765 → 1.363694 seconds, excluding mandatory server READY/startup and +shutdown. Execution has no demonstrated improvement. All four complete authored +image inventories pass exact byte parity and the retained native physical input +qualification; no new native CP timing is claimed. Other workloads and scaling +remain unmeasured for this change. diff --git a/benchmark/results/perf_schema_source_readiness_20261003/receipt.json b/benchmark/results/perf_schema_source_readiness_20261003/receipt.json new file mode 100644 index 0000000000..817e85e994 --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/receipt.json @@ -0,0 +1,261 @@ +{ + "status": "ALL4_ORDINARY_AUTHORED_IMAGES_EXACT_AND_RETAINED_NATIVE_INPUT_PARITY_PASS", + "case": "ExampleIlluminationCorrection_Example3", + "mode": "1w_1t", + "cpu": 5, + "candidate_openhcs_commit": "7258826d7198f2bb7010763276dba02320d62142", + "variants": [ + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/openhcs/processing/backends/lib_registry/registry_service.py", + "baseline_revision": "4eaacf17e5de3dba59ec217f133fa6e1821c8641", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing", + "relative": "openhcs/processing/backends/lib_registry/registry_service.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-B.source", + "A_sha256": "73e0d41f98d7ea5bb3fe076b97b97d9980b621c54f2d23b808550e00bae481d4", + "B_sha256": "d9e6e30b627fae7b01e08872c58ad26ee060919b6b6bd9fd9174223f2ec5a95b" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect/src/python_introspect/signature_analyzer.py", + "baseline_revision": "13f6ff3b43aaa430f5e771848f08d99faf941158", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect", + "relative": "src/python_introspect/signature_analyzer.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-B.source", + "A_sha256": "51f4729861062ab857153c57d64b01bd7b0f47daa6f18fc4443819fb75c77b45", + "B_sha256": "7e25a0355159c86d361a21a7afc449ec758cbf8a32e9b72d06de66a924f04159" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState/src/objectstate/lazy_factory.py", + "baseline_revision": "b0004daf3223326ac78689ba7b2e68abc18ae298", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState", + "relative": "src/objectstate/lazy_factory.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-B.source", + "A_sha256": "f81ce722695dbab98fd02658f2015e5cb93c1e1e69547d24249e62d896e2a828", + "B_sha256": "90637bdad398bae1348168e62533d69618ec965341b1b27057292e709c2a1f85" + } + ], + "means": { + "A": { + "compile_seconds": 0.9621515274047852, + "execute_seconds": 0.38265538215637207, + "total_seconds": 1.693764767376706 + }, + "B": { + "compile_seconds": 0.5876049995422363, + "execute_seconds": 0.40058863162994385, + "total_seconds": 1.3636944827158004 + } + }, + "saved_seconds": { + "compile_seconds": 0.37454652786254883, + "execute_seconds": -0.017933249473571777, + "total_seconds": 0.3300702846609056 + }, + "runs": [ + { + "label": "A", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.9799041748046875", + "prepare_seconds": "0.0", + "execute_seconds": "0.38725757598876953", + "total_seconds": "1.7173671890050173", + "wells_per_second": "2.5822606502835725", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1011.84765625", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "a37695e774a71b04c3e2a390df76245c973458380fca25a9aac513dbf5e8be8e" + } + ], + "receipt_sha256": "6805ee89e2872aea6199e959e53b956ef144e2ff82516fbf6ac68042450d6fd2", + "source_freeze_sha256": "d71dcaef08a9eb2ac12a00b5e004aedfcd9b9606d860e6a8bca5da73baa9900b" + }, + { + "label": "B", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.593247652053833", + "prepare_seconds": "0.0", + "execute_seconds": "0.42087531089782715", + "total_seconds": "1.4185294113121927", + "wells_per_second": "2.3760006208650304", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1043.54296875", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "3c135c9fa72e1a8ec100dca796e96c7f6721cccfab82316b3836236b6b4e75b0" + } + ], + "receipt_sha256": "c8f76ce4b6acaa60f5bbfe13c5a5ca924e299d5c8834e249f757dd53adc434c7", + "source_freeze_sha256": "ac0a8d0ad5938927dbddf01b937ef15387a15c6701d87d292ed486fbcbec7870" + }, + { + "label": "B", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.5819623470306396", + "prepare_seconds": "0.0", + "execute_seconds": "0.38030195236206055", + "total_seconds": "1.3088595541194081", + "wells_per_second": "2.629489524807818", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1043.19140625", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "3c135c9fa72e1a8ec100dca796e96c7f6721cccfab82316b3836236b6b4e75b0" + } + ], + "receipt_sha256": "8a7f0ecb3274b47312e8ee1431b4bd75f9aaba713fd76d3e9ec28ab038aa93da", + "source_freeze_sha256": "d6608b629bb37ecb89d30797258d674d816857671e77fcf580bec3d62910a7a2" + }, + { + "label": "A", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.9443988800048828", + "prepare_seconds": "0.0", + "execute_seconds": "0.3780531883239746", + "total_seconds": "1.6701623457483947", + "wells_per_second": "2.6451304495891326", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1012.59375", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "a37695e774a71b04c3e2a390df76245c973458380fca25a9aac513dbf5e8be8e" + } + ], + "receipt_sha256": "4a804982debefcd6cd075ac16520e85036281b3eddd78cabaf86f41f2841b4c6", + "source_freeze_sha256": "6b40a27bf359bb67febc0af967221829a7719cac0d965704609c0d36991f608e" + } + ], + "tests": { + "python_introspect": 136, + "objectstate_lazy_factory": 25, + "openhcs_readiness": 119 + }, + "upstream_prs": [ + "https://github.com/OpenHCSDev/python-introspect/pull/6", + "https://github.com/OpenHCSDev/ObjectState/pull/9" + ], + "science_runs": [ + { + "label": "A", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-A", + "receipt_sha256": "6805ee89e2872aea6199e959e53b956ef144e2ff82516fbf6ac68042450d6fd2", + "scientific_npy_bytes": { + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6", + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + }, + { + "label": "B", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-B", + "receipt_sha256": "c8f76ce4b6acaa60f5bbfe13c5a5ca924e299d5c8834e249f757dd53adc434c7", + "scientific_npy_bytes": { + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a", + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + }, + { + "label": "B", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-B", + "receipt_sha256": "8a7f0ecb3274b47312e8ee1431b4bd75f9aaba713fd76d3e9ec28ab038aa93da", + "scientific_npy_bytes": { + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6", + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + }, + { + "label": "A", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/3-A", + "receipt_sha256": "4a804982debefcd6cd075ac16520e85036281b3eddd78cabaf86f41f2841b4c6", + "scientific_npy_bytes": { + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a", + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + } + ], + "scope": "Four ordinary uninstrumented public Illumination runs ABBA, CPU5 1w_1t INLINE, default OUTCOMES and memory observer. Server READY preparation/startup and shutdown excluded. A is exact prior source bytes for three files; dependency Git HEADs identify containing checkout, file hashes identify actual baseline contents. Only source lookup, existing registered type projection and registry readiness differ. No scientific processing or numerical changes. No new native CP execution.", + "limits": [ + "Two observations per variant; no execution speedup is claimed.", + "Authored workload exports two images and no tables.", + "Native comparison is retained matched CP4.2.8.1 evidence, not a fresh native run.", + "Ordinary pipeline total excludes mandatory server startup and shutdown." + ], + "artifact_hashes": { + "freeze.json": "2c5d9272f1693e1da821ad45c6db3190cd046f12f446142f9c263611f8a81da8", + "observations.json": "aa72dc885608831f3bbc21efc62f2739d86ec74d64f5ae56d0ee27cd129792fd", + "science.json": "918c36612ec20738ad2d2760da85c9e9be07e81d4cd4bf0f60ec91dfcf906488", + "controller.py": "6fcf1cb1008ac32ee3ceaff8fd61f6db1fdba87544e1ffee1e4b3d484ce30b90", + "validate_science.py": "86fd34f0ca552309a453c5c169a97b949304aa37fb642a66972f4d12f9940684" + }, + "authoritative_artifacts": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1", + "remaining_work": "No execution speedup claimed. Runtime plumbing, other workloads, fresh native/scaling sweeps and figures remain outstanding." +} diff --git a/benchmark/results/perf_schema_source_readiness_20261003/recipes/controller.py b/benchmark/results/perf_schema_source_readiness_20261003/recipes/controller.py new file mode 100644 index 0000000000..effe202bcd --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/recipes/controller.py @@ -0,0 +1,136 @@ +"""Ordinary public ABBA; only three owned source files vary between terminated runs.""" +import csv +import fcntl +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import time + +HERE = Path(__file__).parent +SOURCE = Path('/home/ts/code/projects/openhcs-shared-runtime-plumbing') +MAIN = Path('/home/ts/code/projects/openhcs') +PYTHON = MAIN / '.venv/bin/python' +CASE = 'ExampleIlluminationCorrection_Example3' +VARIANTS = json.loads((HERE / 'variants.json').read_text()) + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + +ordinary = load('ordinary_schema_source', '/tmp/run_integrated_fields_ordinary_abba_20261002.py') +base = load('schema_source_inputs', '/var/tmp/run_narrow450_runtime_phase_probe_20261002.py') +base.SOURCE = SOURCE +base.CASES = (CASE,) +sparse = load('schema_source_sparse', '/var/tmp/openhcs_measurement_sparse_inventory_v2_20261003.py') + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + +def git(repo, *args): + return subprocess.check_output(['git', '-C', str(repo), *args], text=True).strip() + +def environment(): + env = ordinary.environment(SOURCE) + for key in tuple(env): + if key.startswith(('OPENHCS_DIAGNOSTIC_', 'OPENHCS_CAPTURE_CELLPROFILER_FIXTURES_', 'OPENHCS_PROFILE_FUNCTION_RUNTIME')) or key == 'OPENHCS_WORKER_PROFILE_DIR': + del env[key] + env.update(PYTHONDONTWRITEBYTECODE='1', TMPDIR=str(HERE / 'tmp')) + return env + +def sources(): + dependencies = {} + for line in git(SOURCE, 'config', '--file', '.gitmodules', '--get-regexp', 'path$').splitlines(): + _, relative = line.split(maxsplit=1) + repo = SOURCE / relative + dependencies[relative] = {'head': git(repo, 'rev-parse', 'HEAD'), 'status': git(repo, 'status', '--porcelain'), 'files': base.tracked_source_hashes(repo)} + return {'head': git(SOURCE, 'rev-parse', 'HEAD'), 'status': git(SOURCE, 'status', '--porcelain'), 'files': base.tracked_source_hashes(SOURCE), 'dependencies': dependencies, + 'native_binaries': {r: sha(SOURCE/r) for r in ordinary.NATIVE}} + +def select(label): + # This checkout is owned, and the source files are not being edited by agents. + # No live server may remain while its source is changed. + for entry in VARIANTS: + target = Path(entry['path']) + assert target.is_relative_to(SOURCE) + current = sha(target) + assert current in {sha(entry['A']), sha(entry['B'])}, 'Unexpected source edit' + target.write_bytes(Path(entry[label]).read_bytes()) + +def assert_endpoint_terminal(receipt): + import psutil + endpoint = receipt['endpoint_provenance'] + pid = endpoint['endpoint_pid'] + try: + process = psutil.Process(pid) + assert abs(process.create_time() - endpoint['endpoint_create_time_epoch_seconds']) > 0.1 or not process.is_running() or process.status() == psutil.STATUS_ZOMBIE, 'Measured endpoint remains live' + except psutil.NoSuchProcess: + pass + +def main(): + assert shutil.disk_usage('/var/tmp').free >= 512 * 1024**2 + assert shutil.disk_usage('/home').free >= 2 * 1024**3 + assert not (HERE / 'freeze.json').exists() + (HERE / 'tmp').mkdir() + env = environment() + original = sources() + assert not original['status'], 'Commit source before measurement' + physical = base.physical_input_hashes() + sparse_state = sparse.sparse_source_inventory(SOURCE) + installed = subprocess.check_output([str(PYTHON), '-I', '-m', 'pip', 'freeze', '--all'], text=True) + imports = subprocess.check_output([str(PYTHON), '-c', 'import openhcs,objectstate,python_introspect,json;print(json.dumps({m.__name__:m.__file__ for m in (openhcs,objectstate,python_introspect)}))'], env=env, cwd=str(HERE/'tmp'), text=True) + for path in json.loads(imports).values(): + assert Path(path).is_relative_to(SOURCE), path + freeze = {'candidate': original, 'physical_inputs': physical, 'sparse_inventory': sparse_state, 'installed_freeze': installed, 'python_sha256': sha(PYTHON.resolve()), 'controller_sha256': sha(__file__), 'environment': env, 'imports': json.loads(imports), + 'variants': [{**r, 'A_sha256': sha(r['A']), 'B_sha256': sha(r['B'])} for r in VARIANTS], + 'scope': 'Four ordinary uninstrumented public Illumination runs ABBA, CPU5 1w_1t INLINE, default OUTCOMES and memory observer. Server READY preparation/startup and shutdown excluded. A is exact prior source bytes for three files; dependency Git HEADs identify containing checkout, file hashes identify actual baseline contents. Only source lookup, existing registered type projection and registry readiness differ. No scientific processing or numerical changes. No new native CP execution.'} + (HERE/'freeze.json').write_text(json.dumps(freeze, indent=2)+'\n') + rows = [] + lock = Path('/tmp/openhcs-benchmark-xdg-cache/openhcs/official30-runtime.lock') + lock.parent.mkdir(parents=True, exist_ok=True) + try: + with lock.open('a+') as lease: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + for index, label in enumerate(('A', 'B', 'B', 'A')): + select(label) + before = sources() + dest = HERE / f'{index}-{label}' + assert not dest.exists() + command = ['/usr/bin/taskset', '-c', '5', str(PYTHON), '-c', ordinary.BOOTSTRAP, str(SOURCE/'scripts/benchmark_cppipe_well_throughput.py'), '--manifest', str(SOURCE/'benchmark/manifests/official30_portable_axis1.json'), '--mode', '1w_1t', '--case', CASE, '--output-dir', str(dest/'public')] + record = {'index': index, 'label': label, 'source': before, 'command': command} + (HERE/f'{index}-{label}-source.json').write_text(json.dumps(record, indent=2)+'\n') + print(json.dumps({'status': 'RUNNING', 'index': index, 'label': label}), flush=True) + started = time.monotonic() + with (HERE/f'{index}-{label}.log').open('x') as stream: + result = subprocess.run(command, cwd=str(HERE/'tmp'), env=env, stdout=stream, stderr=subprocess.STDOUT, timeout=900) + assert sources() == before, 'Source/dependency/native bytes changed during run' + assert base.physical_input_hashes() == physical + assert subprocess.check_output([str(PYTHON), '-I', '-m', 'pip', 'freeze', '--all'], text=True) == installed + assert sha(__file__) == freeze['controller_sha256'] + measured = list(csv.DictReader((dest/'public/well_throughput.csv').open())) + assert result.returncode == 0 and len(measured) == 1 and measured[0]['status'] == 'success' and measured[0]['successful_wells'] == '1' and measured[0]['execution_route'] == 'ordinary-zmq-outcomes-v1' + receipt_path, = (dest/'public'/CASE/'wells_1/workers_1/ordinary_run_evidence').glob('*/measured_pipeline_receipt.json') + receipt = json.loads(receipt_path.read_text()) + assert receipt['observation_export_scope'] == 'outcomes' and receipt['observed_axis_count'] == receipt['expected_axis_count'] == 1 + assert_endpoint_terminal(receipt) + assert sum(p.stat().st_size for p in dest.rglob('*') if p.is_file()) < 128 * 1024**2, 'Output cap' + rows.append({'label': label, 'directory': str(dest), 'seconds_including_startup_for_audit_only': time.monotonic()-started, 'public_rows': measured, 'receipt': str(receipt_path), 'receipt_sha256': sha(receipt_path), 'source_freeze_sha256': sha(HERE/f'{index}-{label}-source.json')}) + (HERE/'observations.json').write_text(json.dumps({'rows': rows, 'science': 'PENDING', 'scope': freeze['scope']}, indent=2)+'\n') + print(json.dumps({'status': 'RUN_COMPLETE', 'label': label, 'timings': {k: measured[0][k] for k in ('compile_seconds', 'execute_seconds', 'total_seconds')}}), flush=True) + finally: + select('B') + assert sources() == original + sparse.validate_sparse_source_inventory(SOURCE, sparse_state) + from statistics import mean + means = {label: {key: mean(float(row[key]) for r in rows if r['label']==label for row in r['public_rows']) for key in ('compile_seconds', 'execute_seconds', 'total_seconds')} for label in ('A', 'B')} + result = {'means': means, 'saved_seconds': {key: means['A'][key]-means['B'][key] for key in means['A']}, 'rows': rows, 'science': 'PENDING', 'scope': freeze['scope']} + (HERE/'observations.json').write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps({'status': 'ABBA_MEASUREMENT_COMPLETE_SCIENCE_PENDING', 'means': means, 'saved_seconds': result['saved_seconds']}), flush=True) + +if __name__ == '__main__': + main() diff --git a/benchmark/results/perf_schema_source_readiness_20261003/recipes/validate_science.py b/benchmark/results/perf_schema_source_readiness_20261003/recipes/validate_science.py new file mode 100644 index 0000000000..fcbe65f2a6 --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/recipes/validate_science.py @@ -0,0 +1,88 @@ +"""Strict complete authored image inventory and retained native input qualification.""" +import openhcs +from pathlib import Path +import hashlib +import json + +from benchmark.adapters.openhcs import _strict_cellprofiler_runtime_equivalence_policy +from benchmark.matched_cellprofiler_batch import _require_compared_output_inventory +from openhcs.core.equivalence.comparison import runtime_image_differences, runtime_table_differences +from openhcs.core.equivalence.outputs import RuntimeOutputSnapshot +from openhcs.core.runtime_exports import RuntimeExportObservation +from openhcs.core.virtual_workspace_metadata import METADATA_CONFIG, VirtualWorkspaceSourceProjectionEntries +from openhcs.core.source_projection import SourceProjectionSet + +HERE = Path(__file__).parent +REFERENCE = HERE.parent/'illumination-compiler-ledger-v4' +REFERENCE_SCIENCE = HERE.parent/'illumination-compiler-ledger-v4-science.json' +NATIVE = Path('/var/tmp/openhcs-current-illumination3-native-timing-science-v1-20261002.json') +CASE = 'ExampleIlluminationCorrection_Example3' + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + +def read(directory): + receipt_path, = (directory/'public'/CASE/'wells_1/workers_1/ordinary_run_evidence').glob('*/measured_pipeline_receipt.json') + receipt = json.loads(receipt_path.read_text()) + assert receipt['observation_export_scope'] == 'outcomes' + assert receipt['observed_axis_count'] == receipt['expected_axis_count'] == 1 + assert sha(receipt['observation_export_path']) == receipt['observation_export_sha256'] + assert sha(receipt['results_summary_path']) == receipt['results_summary_sha256'] + root, = map(Path, receipt['output_roots']) + assert root.resolve().is_relative_to(directory.resolve()) + files = frozenset(p for p in root.rglob('*') if p.is_file()) + exports = RuntimeExportObservation.from_output_roots((root,)) + snapshot = RuntimeOutputSnapshot.from_export_observation(exports, source_workspaces=(root,)) + return receipt_path, receipt, root, files, exports, snapshot + +def main(): + assert not (HERE/'science.json').exists() + obs = json.loads((HERE/'observations.json').read_text()) + assert len(obs['rows']) == 4 and [r['label'] for r in obs['rows']] == ['A','B','B','A'] + qualification = json.loads(REFERENCE_SCIENCE.read_text()) + assert qualification['status'] == 'CURRENT_BB9_DIAGNOSTIC_FULL_AUTHORED_IMAGE_INVENTORY_BYTE_NATIVE_INPUT_TRANSITIVE_PASS' + native = json.loads(NATIVE.read_text()) + assert native['status'] == 'PASS_AUTHORED_NONEMPTY_SAVED_IMAGE_SCIENCE_AND_MATCHED_FRESH_NATIVE_TIMING' + for role in ('native_report','source_environment_input_witness','saved_image_science_receipt'): + assert sha(native[role]) == native[role+'_sha256'] + ref = read(REFERENCE) + assert sha(ref[0]) == qualification['current_receipt_sha256'] + assert {str(p.relative_to(ref[2])): sha(p) for p in ref[3] if p.suffix == '.npy'} == qualification['scientific_npy_bytes'] + before = {str(p): sha(p) for p in ref[3]} + policy = _strict_cellprofiler_runtime_equivalence_policy() + results = [] + freeze = json.loads((HERE/'freeze.json').read_text()) + cpp = freeze['physical_inputs']['cppipe_path'] + assert cpp['path'] == native['source_cppipe'] and list(cpp['files'].values()) == [native['source_cppipe_sha256']] + environments = [] + for run in obs['rows']: + actual = read(Path(run['directory'])) + assert sha(actual[0]) == run['receipt_sha256'] + before.update({str(p): sha(p) for p in actual[3]}) + td = runtime_table_differences(ref[5].tables, actual[5].tables, policy) + im = runtime_image_differences(ref[5].images, actual[5].images, policy) + assert not td and not im, (td, im) + assert not actual[5].tables and len(actual[5].images) == len(ref[5].images) == 2 + _require_compared_output_inventory(reference_files=ref[3]-frozenset(METADATA_CONFIG.managed_paths(ref[2])), candidate_files=actual[3], reference_exports=ref[4], candidate_exports=actual[4], reference_snapshot=ref[5], candidate_snapshot=actual[5], candidate_managed_files=frozenset(METADATA_CONFIG.managed_paths(actual[2]))) + images = {str(p.relative_to(actual[2])): sha(p) for p in actual[3] if p.suffix == '.npy'} + assert images == qualification['scientific_npy_bytes'] and len(images) == 2 + workspace = Path(actual[1]['execution_plate_id']) + subdirs = json.loads((workspace/'openhcs_metadata.json').read_text())['subdirectories'] + entries = tuple(entry for s in subdirs.values() for entry in VirtualWorkspaceSourceProjectionEntries.from_subdirectory(s).entries.values()) + projections = SourceProjectionSet(entries) + assert len(projections.plane_projections) == 1 and not projections.artifact_projections + assert all(not p.ref.source_axis_indices for p in projections.plane_projections) + selected = {str(Path(p.ref.backend_address).resolve(strict=True)): sha(p.ref.backend_address) for p in projections.plane_projections} + assert selected == native['physical_source_sha256'] + assert actual[1]['global_config_source_sha256'] == ref[1]['global_config_source_sha256'] + assert actual[1]['endpoint_provenance']['client_openhcs_file'] == str(Path(freeze['imports']['openhcs'])) + environments.append(actual[1]['server_environment']) + results.append({'label': run['label'], 'directory':run['directory'], 'receipt_sha256':sha(actual[0]), 'scientific_npy_bytes':images, 'physical_selected_inputs':selected, 'image_differences':im, 'table_differences':td, 'full_physical_inventory':'PASS'}) + assert all(e == environments[0] for e in environments) + assert all(sha(p) == digest for p,digest in before.items()) + result = {'status':'ALL4_ORDINARY_AUTHORED_IMAGES_EXACT_AND_RETAINED_NATIVE_INPUT_PARITY_PASS', 'runs':results, 'means':obs['means'], 'saved_seconds':obs['saved_seconds'], 'all_before_after_output_sha256':before, 'server_environment':environments[0], 'freeze_sha256':sha(HERE/'freeze.json'), 'reference_science_sha256':sha(REFERENCE_SCIENCE), 'native_receipt_sha256':sha(NATIVE), 'validator_sha256':sha(__file__), 'limits':['Two observations per variant; no execution speedup is claimed.', 'Authored workload exports two images and no tables.', 'Native comparison is retained matched CP4.2.8.1 evidence, not a fresh native run.', 'Ordinary pipeline total excludes mandatory server startup and shutdown.']} + (HERE/'science.json').write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps({'status':result['status'], 'means':obs['means'], 'saved_seconds':obs['saved_seconds'], 'science_sha256':sha(HERE/'science.json')})) + +if __name__ == '__main__': + main() diff --git a/benchmark/results/perf_schema_source_readiness_20261003/recipes/variants.json b/benchmark/results/perf_schema_source_readiness_20261003/recipes/variants.json new file mode 100644 index 0000000000..3cbd1f0ed4 --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/recipes/variants.json @@ -0,0 +1,26 @@ +[ + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/openhcs/processing/backends/lib_registry/registry_service.py", + "baseline_revision": "4eaacf17e5de3dba59ec217f133fa6e1821c8641", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing", + "relative": "openhcs/processing/backends/lib_registry/registry_service.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-B.source" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect/src/python_introspect/signature_analyzer.py", + "baseline_revision": "13f6ff3b43aaa430f5e771848f08d99faf941158", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect", + "relative": "src/python_introspect/signature_analyzer.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-B.source" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState/src/objectstate/lazy_factory.py", + "baseline_revision": "b0004daf3223326ac78689ba7b2e68abc18ae298", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState", + "relative": "src/objectstate/lazy_factory.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-B.source" + } +] diff --git a/docs/plans/runtime_plumbing_frontier_20261003.rst b/docs/plans/runtime_plumbing_frontier_20261003.rst new file mode 100644 index 0000000000..3cbe1622bd --- /dev/null +++ b/docs/plans/runtime_plumbing_frontier_20261003.rst @@ -0,0 +1,387 @@ +Runtime plumbing frontier: current measurements and rejected routes +================================================================= + +This checkpoint supports PR #394 and issue #496. It records measurements and +remaining investigation, not a new production optimization. Main #498 was +normally merged and pushed at ``093c2f42d1235b5e218800f57937b6a52b4f5255``. +That transition changed documentation only; production sources remain identical +to the earlier native-qualified integrated source. + +Timing boundaries +----------------- + +Every run uses the original public benchmark driver, one worker and one CPU +thread, default OUTCOMES completion and the ordinary memory observer. Mandatory +registry/library/kernel preparation completes before server READY. Server +startup, preparation and shutdown are outside pipeline clocks. Execution excludes +compilation; total includes compilation and completion. Single-worker execution +is inline. The retained native invocation clock excludes import/JVM/pipeline +loading and includes prepare, modules, post-run and closure. + +Diagnostic nested exclusive clocks partition their original execution root. +They include observer overhead and required work. A callable boundary includes +its filtering, conversion and metadata behavior; it is not a pure-kernel clock. +Inclusive descendants cannot be added to their ancestor. Correct output bytes +do not establish performance improvement, input immutability or callback purity. + +Fresh ordinary Speckles +----------------------- + +Two independent uninstrumented invocations on the current source produced +execution times of 1.199828148s and 1.159908533s. Their means are: + +* Execution: 1.179868340s. +* Compilation: 1.067295671s. +* Total: 2.514483608s. + +Both complete five-file inventories, including all three scientific CSVs, +are byte identical to the qualified baseline. Physical images, pipeline, +configuration, server environment, dependencies and native evidence are joined +to the original receipts. Compared descriptively with the retained native +invocation mean of 1.915068373s, execution is 1.6231x faster and total is 0.7616x +as fast. The execution gap to 2x is 0.222334154s. Native was not rerun for this +checkpoint; two observations do not establish statistical significance or a +causal source improvement. + +The earlier and current pipelines both invoke IdentifyPrimaryObjects twice. +Their relevant source bodies and settings are identical. The prior slower +timing therefore has no established code-change explanation. The current +ordinary progress spans put about 0.820924s in those two steps, approximately +70 percent of execution. + +Actual declumping inputs and boolean output oracles were separately captured +under unchanged storage limits. The original two stage bodies took 0.070122s +and 0.001791s, totaling 0.071913s. Capture overhead was excluded from these +stage clocks. Even eliminating that stage cannot bridge the measured gap, so +the seed optimization route is stopped before a candidate or replay is built. +The original diagnostic wrapper import failure is retained separately; the +successful harness passed controls for all five actual installed hooks. + +Current 3D plumbing +------------------- + +An unchanged 24-boundary diagnostic on the current source completed with both +profilers disabled. Its execution root is 8.958889063s, partitioned into +4.163085915s at 211 callable boundaries and 4.795803148s outside them. The latter +is approximately 53.5 percent of execution. Selected disjoint exclusive terms: + +* Stack loading: 0.965303496s across 32 calls. +* Output saving: 0.608366542s across 28 calls. +* Worker-lane residual: 0.619469857s. +* Image-request construction: 0.390939402s across 30 calls. +* Module-output recording: 0.413856133s across 26 calls. +* Metadata publication: 0.493661132s across five calls. +* Artifact metadata reconciliation: 0.288939269s. +* Validation and unstacking: 0.189130950s across 28 calls. + +There are zero feature-query calls. The complete six-CSV/120-TIFF scientific +bytes match the qualified ordinary baseline, all 128 inventory paths match, +and three physical input volumes, the pipeline and retained native logical +volume/plane witnesses are verified. Only the managed metadata JSON differs, +with both hashes explicitly reported. This is a diagnostic breakdown, not an +accepted ordinary runtime gain. + +The prior ordinary 3D execution of 8.536s versus native 14.396s leaves roughly +1.34s to reach 2x execution. Publication and reconciliation together have only +a 0.7826s diagnostic ceiling before mandatory work. A publication-only change +is therefore insufficient. The investigation must span shared producer/source +derivation across loading, image requests, recording, saving and publication. +The existing nominal authorities are the intended owners; no extra mirror, +global cache, per-function shortcut or delayed per-step visibility is admitted. + +Measurement frontier and architectural constraints +-------------------------------------------------- + +The separate 42-boundary Beginner diagnostic measures a disjoint table +assembly/query/pivot envelope of 1.462782s. Native CSV rendering is another +0.163899s. All nine scientific output files and complete physical inventory +match the native-qualified baseline. Its occurrence graph reaches the original +4,096-event limit and remains RED independently of scientific correctness. +The complete Speckles graph demonstrates repeated access to one table and +object-label vector, but that entire measurement lane is only about 0.038s. +Neither query-only nor the 16-call intensity projection route can close the +dominant gaps, and the measurement fold is not a 3D optimization: that workload +does not perform these queries. + +Existing serialization views have distinct semantics. Raw projection source +metadata and enriched SOURCE_METADATA cannot share one output dictionary. +The transaction overlays raw path records before decoding surviving entries; +decoding overwritten malformed entries earlier would change first errors. +Serializer callbacks, independently mutable outputs, current per-step +publication visibility and concurrent-axis final pruning remain obligations. +A frozen projection declaration does not establish deep immutability of its +image metadata. Ownership proposals remain provisional until those laws and +representative replay show a sufficient end-to-end payoff. + +Retained evidence +----------------- + +The following local receipts contain exact source/controller/environment/input +pins, original commands, file inventories and hashes: + +* ``/var/tmp/openhcs-current-speckles-ordinary-pair-v1-20261003/observations.json``: + SHA-256 ``d08f35061285e8a4d543377c1de7993623654844eddcaea806d269672a0bf4a1``. +* ``/var/tmp/openhcs-speckles-seed-capture-v4-scientific-byte-gate-20261003.json``: + SHA-256 ``4da188db100b9526f18a6a02ce98872dad6e7ee5dca60c1b85e7052e1287e301``. +* ``/var/tmp/openhcs-current-3d-runtime-ledger-v1-scientific-byte-gate-20261003.json``: + SHA-256 ``36a4f2cdcc14baaa38f027a6dc759facd1229cc1f5960ddb063cbe689efda0cf``. +* ``/var/tmp/openhcs-measurement-occurrence-ledger-v1-scientific-byte-gate-v2-20261003.json``: + SHA-256 ``4352fa24e8984e0f4c67587ae674ff28085ed134531dea28fe7d5b8517671f99``. + +PR #394 remains draft. Whole-branch original R0 and the #479 per-file R1 +failures remain explicit and unwaived; installed consumer acceptance, +full-catalog/scaling measurements and fresh figures remain unfinished. +The performance goal remains active. + +Deeper current 3D attribution and producer ownership +--------------------------------------------------- + +Two further diagnostics use clean source ``396379b06769a3f5afe92413b7d9d6a21dd3f66f``. +Its production, test and benchmark driver bytes are unchanged from the preceding +``093c2f42`` source. Both preserve mandatory READY warmup, inline 1w_1t execution, +OUTCOMES export, the normal memory observer and both disabled profilers. Neither +is a new ordinary performance comparison. + +The 36-hook run separates metadata normalization and provenance operations from +their actual nested callers. It observes 29,701 normalization calls outside raw +callable boundaries, totaling 0.317682s exclusive. Ordinary normalization creates +a fresh scalar identity and reuses the cached plane collection; it does not copy +the complete 60-plane lineage. Whole-plane merges and derived naming do rebuild +plane/contributor snapshots. Those operations retain live-mutation and fresh +snapshot obligations. Their observed costs do not admit a standalone fix large +enough to close the 1.338s qualified execution gap. + +Several large isolated stalls prompted a separate 25-hook run with passive GC +callbacks. It does not disable GC, freeze objects, change thresholds, force a +collection or replace existing callbacks. Startup has no active pipeline root. +Its execution root is 8.462928s: 3.803686s within the 211 declared callable +boundaries and 4.659242s outside them. The public diagnostic clocks are execution +8.490154s, compilation 1.824986s and total 11.210223s. GC consumes 0.285505s, +including one 0.196025s generation-two collection during saving. Collection +durations overlap the owner clocks; adding them to plumbing would double count. +GC alone cannot explain or close the remaining gap, so a GC-policy optimization +is rejected. Different diagnostic clocks on identical production bytes are not +evidence of a production gain or regression. + +Both diagnostics match all six CSVs and 120 TIFFs byte for byte against the same +native-qualified baseline, with complete 128-file inventories and physical +input joins. No fresh native repetition is claimed. The 36-hook scientific +output is retained in a lossless, individually hash-verified archive at +``/var/tmp/openhcs-current-3d-owner-ledger-v2-20261003/scientific-output.tar.gz``; +its custody receipt records every original path, mode, size and hash. Later +readers must use or restore that archive rather than assume those extracted +files remain present. The 25-hook output uses the same verified custody protocol +at ``/var/tmp/openhcs-current-3d-gc-ledger-v1-20261003/scientific-output.tar.gz``. +Previous baselines and frozen failures are unchanged. + +Source and consumer audits reject two tempting shortcuts. Artifact-only groups +already return ``NoMainFlowOutput`` and skip unstacking and saving. Canonical CP +publication already reuses the stored runtime artifact payload. Downstream CP +inputs deliberately distinguish stored secondary artifacts from relation-owned +current main flow. The latter has an independent mutable whole buffer and final +filename/source-component context. Repeated cache hits share that buffer. A +direct stored-artifact substitution would change these relationships and, in a +saved determining case, 62 metadata facts. The final named output bundle can +also retain earlier sibling outputs, so the last canonical return roster cannot +stand in for the complete main-flow cohort. + +The remaining structural route belongs to existing ``PatternGroupOutputData``, +``AlignedImageStack``, manifest and runtime-input owners: derive physical plane +views and final context from the exact correlated producer occurrence, preserving +the independent main-flow buffer, rather than repeatedly project and recompose +the whole cohort. The seven marked load/request/record/unstack/save/publication/ +reconciliation spans total 3.432617s in the GC diagnostic. This is an upper +envelope containing mandatory work, not a claimed removable cost. Before a +candidate, representative replay must retain the stored occurrence, actual +merged bundle, projected leaves, final records and cache, filename/path prestates, +and next consumer edge in one graph. No fake processing context, new cache, +canonical-roster shortcut or zero-copy assumption is admitted. + +Additional retained receipts: + +* ``/var/tmp/openhcs-current-3d-owner-ledger-v2-scientific-byte-gate-20261003.json``: + SHA-256 ``df4f6548f4459c6a55ea5dc182ac52a5472eaacace572ae60740a9409377ca89``. +* ``/var/tmp/openhcs-current-3d-gc-ledger-v1-scientific-byte-gate-20261003.json``: + SHA-256 ``d0b8d6af1b8a1777b03624cf6435d67b04746e9477a5cefbc7a962feaa2d667d``. +* ``/var/tmp/openhcs-current3d-canonical-mainflow-CP-owner-review-v1-20261003.json``: + SHA-256 ``9c264eaa2a735388f5694a305dcbf8ff61258ed959adb4819ffacfcb39235bbf``. +* ``/var/tmp/openhcs-current3d-canonical-mainflow-macro-owner-audit-v4-20261003.json``: + SHA-256 ``c589df6f3c28d6d3a3dc668adc696089bc4aeb68aa2d762a0128592e8d67b891``. + +This checkpoint changes documentation only. No optimization, merge, original +R0/R1 waiver, installed acceptance or full-catalog/scaling completion is claimed. + +Current producer/consumer evidence and compilation frontier +---------------------------------------------------------- + +Fresh diagnostics at ``bb9c49c2ad70f1cee49fa0cde47592681edb2be3`` preserve +main ``87d9a99a9``, shared dependency/native/input pins, normal READY warmup, +inline 1w_1t execution, OUTCOMES completion and the normal memory observer. +Their observation clocks are not ordinary performance improvements. + +The bounded V7 capture retains four actual step-13 ImageMath producer and +step-14 Resize load states, totaling 856,419,294 bytes under the unchanged +384 MiB per-item and 1 GiB total limits. Its full six-CSV/120-TIFF parity, +complete scientific inventory, all 180 ordered physical source references, +and three native input-volume witnesses pass. Original step-13 replay matches +all 60 records, named/VFS/cache outputs and the complete 123-array identity, +sharing, dtype, stride and writeability graph. + +Original step-14 replay is RED before cache lookup: the actual producer +manifest lives in module ``_STEP_OUTPUT_MANIFESTS``, a WeakKeyDictionary outside +ProcessingContext transport. Reconstructing it from the separate producer +oracle would fabricate a receiver state and is rejected. Independently, +RuntimeSourceBindingContextCache deliberately drops four derived caches through +its reducer. The actual after-load graph proves the result is the retained +whole image for its exact 60-path cache key, but cold restored clocks cannot +represent warm pipeline work. A future replay must carry actual manifest/cache +owners in the same graph and rekey identity indexes only from real owner +references, or obtain live production confirmation. Original failed artifacts +remain preserved; schema completeness is not complete transaction state. + +A six-group ephemeral layout experiment validates the proposed existing-class +projection ancestor layout, constructor signatures, default factories, frozen +assignment, cooperative projection order, errors and fresh public provenance. +This proves layout feasibility only. Lazy read algorithms, custom subclass +behavior, combined transaction acceptance and sufficient elapsed savings +remain unproved. No new wrapper or cache authority is implemented. + +Illumination remains the weakest known total comparison. Its current V3 +diagnostic records compilation 1.125192s, execution job 0.377852s and total +1.844212s, with both authored NPY files, full inventory and actual physical +TIFF/pipeline/native witnesses matching the retained qualified reference. +The compilation partition places 0.302983s exclusive in dataclass analysis +and 0.285258s in nested documentation extraction: 0.588241s together. Four +client pipeline renders plus three config renders take 0.235231s. The joint +0.823472s envelope contains required work; it is not a saving estimate. +Ten effective-config getters include the schema/state subtree and cannot be +added to it. Completion alone is too small to close the remaining gap. + +The separate V4 diagnostic also passes full science and distinguishes actual +cache topology: compilation has 40 successful first-use misses and 338 hits +across 44 class identities; execution has 94 hits. There are no uncached empty +results. This rejects repeated failing analysis or fresh proxy churn as the +mechanism. Exact runtime class names were not retained because the diagnostic +read a type-owned attribute from the class dictionary; that metadata omission +is explicit. A separate declaration inspection finds 22 registered public/proxy +pairs and 95 differing fields, including Optional annotations and None defaults. +Canonical-class ParameterInfo substitution would change real reset and +reconstruction consumers, so it is rejected. The remaining candidate must +separate reusable declaration work from correctly timed value/default-factory +evaluation using the existing schema and preparation owners. + +Local source-qualified receipts: + +* ``/home/ts/.local/state/openhcs-maintenance/20261003/current-cohort-capture-v7-science-verification-v2.json`` + (SHA-256 ``99f2f830861eb1ecd1492c50ff6bbb2a66172674211e0a925f9f6a85359a1be0``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/current-cohort-v7-original-replay-v3.json``. +* ``/home/ts/.local/state/openhcs-maintenance/20261003/current-cohort-v7-load-owned-role-review.json`` + (SHA-256 ``bd08002bbb535911ba189b1b97bf8cbceb03c938b89c5ec8d0c061e4743735a7``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/projection_ancestor_feasibility_bb9c_v3.json`` + (SHA-256 ``b5ff353e33c70d4066de5315c0bef2046e1e8680056125404a7c393eb2e531ae``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/illumination-compiler-ledger-v3-science.json`` + (SHA-256 ``e042e25aa0fd9e087459c77613150f0db6447e91d0fc943acf2ea638312cc850``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/illumination-schema-cache-topology-v4-20261003.json`` + (SHA-256 ``134244ea39004ccfc86d20fa14d4fdc49e78aa72f4a92349092c6f420c1b89e3``). + +The preceding V3/V4 diagnostics are attribution only. They do not establish a +production saving, new native repetition, original R0/R1 waiver or +full-catalog/scaling completion. + +Registered configuration source readiness +----------------------------------------- + +The decisive source-only prototype found repeated Python 3.10--3.12 class +source lookup walking the complete config module. The existing +``SignatureAnalyzer`` now derives bounded module AST and qualified-class source +views from current immutable source contents. Python 3.13 and newer retain the +native locator. Live annotations, factories, mutable field/inherited docs, +loader invalidation and the original completed value-analysis cache remain at +their existing observation points. Earlier class-identity snapshots failed +real mutation controls and were rejected. + +``LazyDataclassFactory.registered_type_pairs()`` projects the existing registry +without a second roster or constructing values. ``RegistryService`` prepares +both public and proxy declarations after callable hooks and before READY, +including declarations registered after the catalog was cached. Default +factories are not evaluated during source preparation. + +At OpenHCS ``7258826d7198f2bb7010763276dba02320d62142``, four ordinary public +Illumination runs in ABBA order measured: + +.. list-table:: Ordinary 1w_1t mean seconds; two observations per variant + :header-rows: 1 + + * - Phase + - Prior source + - Prepared source + * - Compilation + - 0.962152 + - 0.587605 + * - Execution job + - 0.382655 + - 0.400589 + * - Total pipeline + - 1.693765 + - 1.363694 + +Compilation saves 0.374547s (38.9%); total saves 0.330070s (19.5%). No execution +gain is claimed. All four runs use CPU5, INLINE one-worker execution, normal +OUTCOMES and the default memory observer, without hooks or profilers. Mandatory +server startup/preparation and shutdown are outside total pipeline timing. +Exactly three owned source files vary; immutable before/after hashes identify +actual baseline contents separately from containing dependency Git HEADs. +All source, dependencies, native binaries, environment and physical inputs were +checked before/after each run; the clean candidate was restored afterward. + +All four complete authored output inventories retain both NPY artifacts exactly +and match the qualified physical TIFF/pipeline/native evidence. The workload +authors no tables. Native CellProfiler was not rerun in this experiment; broad +scaling or new native speedup claims are not made. Science receipt SHA-256: +``918c36612ec20738ad2d2760da85c9e9be07e81d4cd4bf0f60ec91dfcf906488``. + +The 136 upstream, 25 existing lazy-factory and 119 OpenHCS readiness/catalog/ +startup tests pass. python-introspect PR #6 and ObjectState PR #9 are merged and +formally close their issues #5 and #8. Their merge commits change no measured +source bytes. OpenHCS wiring is pushed in draft #394; a small main-only readiness +change can be isolated without merging the unresolved broad branch. +Reviewable measurement receipt and recipes are under +``benchmark/results/perf_schema_source_readiness_20261003``; original immutable +outputs/freeze remain under the corresponding maintenance ABBA directory. +Runtime plumbing, full-catalog/scaling measurements, fresh native repetitions +and figures remain outstanding. Original whole-branch R0/R1 failures remain +explicit and unwaived. + +Canonical provenance identity integration +------------------------------------------ + +Main readiness #514 is merged into this branch. The overlapping readiness +regression was removed from the callable ABI suite; the dedicated main test +remains authoritative. Callable canonical-signature preparation still precedes +registered declaration preparation and kernel readiness. + +The existing ``SourceMetadataFields.provenance_identity_items`` now derives its +representation from ``identity_items``. Equal nested field mappings have equal +provenance fingerprints independent of insertion order. Raw wire order, real +value differences, ordered planes and lifetime-selected views remain intact. +Existing birth fingerprints are preserved by pickle/cloudpickle; they are not +recomputed from subsequently mutated public fields. Historical nested-mapping +fingerprints can therefore differ from newly canonical fingerprints. A fresh +same-source capture is required before candidate fingerprint replay. + +The integrated metadata/transport/readiness controls pass 86 tests in 2.40s. +The separate exact PR #494 source qualification passes its unchanged 31 writer +controls plus seven identity controls. Those writer controls cannot collect on +this branch because its independent ``PointROIOutput`` producer is absent; no +installed, native or live viewer acceptance is inferred. The integrated receipt +is ``~/.local/state/openhcs-maintenance/20261003/pr494-identity-owner-scratch/ +integrated-owned-qualification-v1.json``. + +V8's original producer and downstream loader replay now pass complete value, +metadata, manifest and array-alias gates, and the full six-CSV/120-TIFF science +gate passes. Serialization still drops nested metadata views and source-context +derived caches. These scoped clocks are not complete production-warm performance +evidence. The saved downstream cache hit constructs no new provenance objects; +it does not establish the payoff of a broader loading/recording/publication fix. +The next capture must retain actual nested caches in the same graph, under the +existing admission bounds, without rewarming or inventing state. diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst new file mode 100644 index 0000000000..37775f781a --- /dev/null +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -0,0 +1,495 @@ +Live fractional-Z point projection (partial152) +============================================== + +Owner: Planck. Parent retains152/134 continuous source-bearing native acceptance. +No competing production claim found:468 changes tracking.py;404 carries saved +writer proposals/evidence;394 does not change Napari/viewer sources. Shared +materialization source ownership must be coordinated before editing that seam. + +Newest public95 checkpoint: FAILED, not accepted +----------------------------------------------- + +The actual public tiny producer compiled, but execution +9292bfb8-4d2e-4e54-9bcc-5af998083fe5 failed at persisted-image metadata +publication after the point archive was written/streamed. Root435/394 owns +that publication join; neither actual conflicting metadata pair is logged. +The original failure, partial live/reopen journals and targets remain immutable. + +Original PolyStore readback of the actual989B ZIP proves all four source planes, +fractional Z1.5, object7 and explicit2/.65/.65 micrometer calibration intact. +Archive SHA256fd0a372f3100c73da9533b308d44c2f85fa7a86b733e2608ae986135cb4e401a. +Persisted reopening transported geometry without its full source record; +the native point handler narrowed represented Z0..3 to occupied0..2; +native presentation calibrated XY but discarded explicit Z spacing. All9 +original XY/XZ/YZ PNGs were personally opened: rendered transport works, +but automatic completion, full source preservation and calibrated acceptance do +not. No scientist input, installed target or native endpoint was changed here. + +This source checkpoint repairs those disjoint receiving owners. StreamingService +keeps the original source-bound ROI payload. ROIArchiveSourceMetadata owns the +single source-only feature exclusion used by native Points and Shapes; it does +not strip transport source facts. Points keeps the full represented source span +from its original anchor while preserving missing-axis/anchor/fractional guards. +NapariAxisPresentation derives scale, units and world translation together for +Image/Shapes/Points and shared-axis reprojection. Explicit3D spacing calibrates Z; +2D/unspecified calibration does not invent it, and selectors/bands stay unscaled. +No mirrored source domain, codec, registry, publication guard relaxation or +shared Root producer/compiler change. Singer522 retirement methods are untouched. + +Before-edit named AST family:704 production+95 original paired dependency modules, +zero parse omissions; source relationships read semantically, not a global NRA +proof. Original source controls incorrectly asserted occupied0..2; the actual +receiving failure demonstrated their insufficient extent assertion. New source +controls require the complete represented extent and actual reopened metadata. +Calibration controls cover2D/3D/relative/unspecified coordinates, scaled offsets, +native Image/Points world alignment and semantic navigation without a Qt app. +Qualification is pending; no installed or new native acceptance is claimed. +The initial direct-source pytest attempt failed before collection because +vendored dirty ZMQ lacks AckReturnRoute. That failure is retained, not fixed by +editing dependency checkouts. + +Reported technical failure +-------------------------- + +Execution5a4d513e-8f27-463f-8177-c657c5457f44 retained failure during viewer +settlement after persisted measurement/point outputs. Read only native6015 log +lines63..114: display_layer_batch -> display_axis_projection -> existing +NapariPointsLayerDisplayHandler.geometric_component_values rejects: +"Fractional-Z point ROI requires a projected z_index axis." +No scientific images, measurement rows, golden answers or pipeline parameters +were read. No endpoint was contacted and no scientist was sent hints. + +Investigation/acceptance +------------------------ + +Trace original typed PointROIOptions writer, ROIFractionalZ, source-plane +provenance, artifact source emission, live component semantics, and persisted +reopening as one family. Generate an independent tiny source-bearing PURE_3D +fixture; identify configuration/layout misuse versus lost Z domain. Preserve +existing absent-axis/domain guards. No mirrored axis domain, alternate codec, +ad-hoc registration or fabricated source anchor is acceptable. + +AST owner+all-consumer inventory precedes changes. Existing refactor-audit +Package/ParsedModule loader is reused. Scope <=512MiB/one CPU/noSwap,60s/shard; +no native/viewer/catalog/display startup or science. Source checkpoints and +proportionate controls will be recorded here. Installed real-path acceptance +requires a later parent-released terminal94/95/96 slot and is NOT claimed. + +This partial repair must not close152's remaining native orthogonal acceptance +or134's saved-output ownership. Current phase03 installed target stays immutable. + +Source-only determining checkpoint +---------------------------------- + +Production base:1a89e09ebbf2bb744d5104cf6f70de93119f1adf. Ownership branch +fix/live-fractional-z-point-projection-20261002; first checkpoint4c9cd972895c. +Ten determining owner/consumer files compare byte-identical to the readonly +engineering484 ordinary installed target: materialization core/options, +function_artifact_materialization, stream_component_semantics, +roi_point_metadata, roi_source_metadata, source_image_provenance, +viewer_streaming_service, napari_viewer_server and napari_streaming_handlers. +This is not a claim of equality to all current main files. + +The independent executable reproducer is +``docs/validation/synthetic_live_fractional_z_points.py``. It creates only +four own5x7 uint16 source TIFFs (4x5x7 volume), invokes an original declared +PURE_3D NumPy producer, computes a centroid atZ1.5/Y1.5/X2.5, and materializes +its typed MeasurementTable through the original PointROIOptions disk writer. +Raw fixture spacing is2/.65/.65; the archive writer carries original source +provenance, not a newly fabricated physical calibration. No scientific pixels +or measurement rows are consumed. No compilation session, runtime process, +catalog, viewer instance, Qt application or endpoint connection is started. +The typed unused endpoint6199 is only a StreamingViewerSurface declaration. + +Actual original-path observations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Saved archive ROIFractionalZ domain is exactlyZ0,1,2,3, with original paths + and component metadata. Its common scalar address omits varyingZ. +* MaterializationSpec.emitted_source_identities returns no identities for + PointROIOptions. This method describes separately emitted projected planes, + not all source planes represented by a single point archive. +* ArtifactStreamSourceMetadataAuthority.payload_source_identity calls + image_payload_metadata. MeasurementTable owns SourceImageProvenanceFields, + not ImagePayloadMetadataCarrier, so absent context fallback this query + returns empty metadata. No-axis live scope results. A valid common scalar + fallback retains other axes but still excludesZ. +* StreamComponentMessageExtraAuthority scopes component order from those + incomplete items before constructing the domain. The unmodified Points + geometric_component_values rejects both scopes with the exact reported + missing projected-Z error. +* An exact first-source-plane fallback restores only singletonZ0. The same + guard rejects Z1.5 as exceeding that domain. An anchor alone is not a repair. +* Supplying the complete saved source-domain through the existing typed + StreamSourceComponentMetadataItems/from_viewer_surface projection succeeds: + the unmodified original Points guard reports occupiedZ0,1,2. This exercises + the same domain projection used by persisted reopening; it is NOT a native + reopen or live settlement acceptance journey. + +Thus the source-only fixture proves a live producer-to-projection contract gap, +not a need to remove the Points guard. No inference is made about the scientist's +algorithm, detected objects or expected counts. + +Reproduction and bounds +~~~~~~~~~~~~~~~~~~~~~~~ + +The installed control ran in a systemd user scope with MemoryMax512M, +MemorySwapMax0, CPUQuota100%, affinityCPU3 and timeout60s; terminal0. Use the +existing paired Python and readonly engineering484 import root:: + + systemd-run --user --scope --unit=openhcs-points152-final-control07 \ + -p MemoryMax=512M -p MemorySwapMax=0 -p CPUQuota=100% \ + timeout 60s taskset -c 3 env \ + PYTHONPATH=/home/ts/wt/openhcs-issue-batch-20260929/engineering484/installed \ + OPENHCS_CPU_ONLY=true OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 \ + /home/ts/wt/openhcs-paired-raw-installed-parent-20261001/.venv/bin/python -B \ + docs/validation/synthetic_live_fractional_z_points.py \ + --output /home/ts/wt/openhcs-issue-batch-20260929/live-points-152-synthetic07 + +The output directory must not exist; select a distinct owned path for a new +engineering invocation. Original07 artifacts are retained and not overwritten. +Terminal PTY62122, invocationdb70c70201d64334835bd7d02aa6ff47. +Source SHA256a22b227bce171597944785f282f8530fdc8ecc415068d42f5522fbc4ca3a9896. +Archive SHA2568276d6ad1d0ce1444aa17c78b367af9b929b6a6187163c913451c4bcb7b58b09. + +AST and honest failures +~~~~~~~~~~~~~~~~~~~~~~ + +Original refactor-audit ParsedModule/Repository traversal inventories704 +OpenHCS modules plus95 actual paired PolyStore/ZMQRuntime modules,364 named +owner/consumer sites, no parse omissions. Persistent evidence: +``/home/ts/wt/openhcs-issue-batch-20260929/live-points-152-AST-BEFORE03.json``. +This is named-family source evidence, not a complete NRA detector proof or +dynamic resolution/equivalence proof. Retained all-at-once first inventory was +OOM-killed at512MiB; second path-loader trial covered0 OpenHCS modules and was +not accepted. The file-at-a-time third inventory supplies the stated coverage. + +Earlier synthetic harness failures are retained separately:01 fake non-dataclass +surface;02 unsupported MeasurementTable source_voxel_spacing keyword; +03 str rather than Path for original ZIP loader; observation03 mappingproxy +passed to bare JSON rather than existing to_jsonable. These are not product +failures or successful reproductions. Observation04, guard05, complete06 and +complete07 controls terminated0. No production source was changed to pass them. + +Owner decision and remaining closure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +NRA/refactor-audit patterns IDEN-1/2 distinguish represented source span from +number of emitted files; BOUND-2 forbids bypassing the provenance owner; +IMPL-4/12 require an actual declaration-owned implementation used by the live +consumer rather than a new viewer-local fix or duplicate domain builder. + +Root394 head e742defc2e57d576362a2c9115e1ca6429a3847f owns shared materialization +core and function_artifact_materialization/source plumbing. Parent explicitly +holds those files pending owner release. No production edit is made here. +The exact proposed seam is posted on394 (comment5963068617), but affirmative +release is not yet recorded at this checkpoint. + +Proposed bounded closure: PointROIOptions declares represented stream source +identities through the existing options/spec family; its point writer preserves +the full original archive metadata and supplies an address derived from the +exact first provenance plane. Migrate the two original emission-query consumers +to the truthful stream-domain contract, retaining other writer behavior. Never +pretend PointROIOptions emits multiple plane files; never set a syntheticZ +on common metadata, add a parallel source-span store or weaken native guards. +Root may implement that narrow producer seam instead and coordinate its live +consumer; Planck owns this494 investigation and regression fixture. + +Before acceptance: coordinated coherent producer/consumer patch, original +2D/3D/multiple-artifact/bad-domain controls, after-inventory deletion evidence, +and parent-released installed continuous source-bearing live/reopen native +journey. No source fix, scientist rerun or native acceptance is claimed yet. + +Root394 integration review at4110503 +------------------------------------ + +Reviewed the pinned current Root394 head +4110503cb543ed525d787f678abdf6efe374b8b7 against production base1a89e09e. +Of the ten determining files, only source_image_provenance.py, +function_artifact_materialization.py and materialization/core.py changed. +The seven remaining files, including PointROIOptions, the Points guard, +ROIFractionalZ, saved reopening and component scoping, are unchanged. +This is a source diff/semantic review, not a new scan or execution claim. + +The required integration adjustment is to use Root's MaterializationBatch +render/save and SavedMaterializationOutputs ownership. Keep the exact typed +point Output and its complete archived provenance flowing through that batch; +do not restore deleted output-group rendering or re-derive publication paths +from logical records. MaterializedRuntimeArtifact now observes actual successful +backend outputs, so the point repair must not add a competing save/path ledger. + +SourceImageIdentity and common metadata now delegate snapshots/composition to +SourceMetadataFields, preserving OwnedSourceMetadataFields lifetime. Derive +represented identities and the anchor through existing provenance methods, +without flattening those records to a separate raw mapping. The source-provenance +initializer now accepts the raw InitVar tuple and decodes at its owner; +MeasurementTable's consumer is migrated accordingly. The published fixture uses +the original constructor, not a direct call to that internal initializer. + +The remaining defect seam is unchanged: artifact backend kwargs are still built +before prepare_materialization renders its batch, using the same +ArtifactStreamSourceMetadataAuthority.metadata_items and +MaterializationSpec.emitted_source_identities logic. Root's new save ledger does +not provide a represented PointROI Z domain to that earlier live projection. +Both full source span and exact original anchor obligations remain. + +No production edits, tests, scans, installs or endpoint requests were performed +for this review. The original8c1e3160d fixture/control remains published and +qualified only at its original immutable installed484 strength, not executed +against Root4110503. Independent consumer diagnosis is complete. Named dependency: +Root394's affirmative shared-seam release or coordinated producer implementation. +The hold and parent-owned later installed native acceptance remain in force. + +Released producer implementation checkpoint +------------------------------------------- + +Root394 explicitly released the narrow options/spec declaration, typed point +output/writer anchor and original artifact source-query migration at +396379b06769a3f5afe92413b7d9d6a21dd3f66f. The source branch is normally merged +with main9531d594 (including504); the five foreign dirty gitlinks are untouched. +Source provenance construction, compiler and generic load/stack remain Root's +ownership and are not edited. No scientific catalog/endpoint is contacted. + +PointROIOptions now owns both typed measurement selection and the represented +stream source-plane domain. FileOutputOptions carries the previous projected +plane implementation; MaterializationSpec delegates rather than deciding which +writer qualifies. The misleading emitted_source_identities entrypoint is +deleted, with its artifact-metadata consumer and two test consumers migrated to +stream_source_identities. Number of files and represented source planes remain +distinct facts (IDEN-1/2, IMPL-4, BOUND-2). + +The original point writer returns PointROIOutput, an Output subtype whose +required original metadata retains the full archive provenance. Its stream +address derives from provenance.for_source_plane(0), without overwriting common +archive metadata or adding an anchor store. Existing Output replacement and +viewer backend batching consume that property normally. All fractional-Z +guards and persisted metadata formats remain unchanged. + +Fresh original refactor-audit file-at-a-time inventory is retained in +``live-points-152-owner-before09.json`` under the parent issue-batch root. +The preceding08 invocation rejected an invalid git revision WORKTREE before +reading source; it is not an accepted census. This is an implemented source +checkpoint awaiting bounded original-writer/projection regression verification; +it is not installed live settlement or native reopening acceptance. + +Bounded original-writer/projection controls +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``tests/unit/test_live_fractional_z_point_source.py`` exercises seven controls +through a real declared PURE_3D tiny producer, the original point/CSV writers, +disk ZIP save/load, artifact source query, original viewer-backend kwargs and +unchanged Points geometric guard. Six positive cases cover singletonZ0, +four-planeZ0..3 andZ10..13, with direct and selected table payloads. A combined +CSV+point declaration retains one point archive while representing every source +plane. Exact first path/Z anchor and full unmodified archived provenance pass; +replacement preserves the nominal point output. Every positive also rejects +an out-of-domain coordinate through the original guard. Wrong payload type is +rejected by the original point declaration, used by writer and query alike. + +Control11 passed7 in11.16s, terminal0, maxRSS446292KiB, swaps0, scope512MiB, +noSwap/oneCPU/timeout60s. No native, viewer, catalog, Qt application or connection +was started. It uses an exported3a11b source snapshot with paired dependencies, +not a new installed target or environment. The reused native tabular binary is +from retained installed02; its original C++ source compares identical. The +snapshot avoids activating the five foreign dirty external checkouts and is +NOT an installed wheel/native acceptance claim. + +Original failed controls remain:08 selected the live checkout via working +directory and rejected stale-loaded externals;09 lacked pytest's configured +test-plugin import root;10 used a wrong ``path`` keyword in the test readback. +The corrected11 uses the original ``file_path`` contract. No production guard +or input assertion was weakened to pass these harness failures. + +Fresh after-inventory10 covers704 OpenHCS and95 paired PolyStore/ZMQ modules, +667 named owner/consumer sites, zero parse omissions (before09:661 sites). +The old emitted_source_identities declaration and production/test consumers +are gone; immutable original defect reproducer and historical receipts retain +that name as evidence. This named AST census is not a full NRA detector/proof. + +Regression and current integration disposition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Regression12 passed25 selected existing controls in9.37s, terminal0, +maxRSS452120KiB, swaps0, same512MiB/noSwap/oneCPU/60s scope. These cover original +point ZIP reopening and rejection, generic source-identity image files, file +bundles, TIFF stream/projection, singleton ROI metadata and filename identity. +47 unrelated cases were deselected; two pre-existing pytest async-configuration +warnings remain. This plus7 new controls is32 passes, not a full test suite or +installed native settlement acceptance. All owned scopes are terminal/inactive. + +After the fresh H002 stop was reported, determining remote heads were checked: +main9531d594,494 at91a374edd and Root394 still396379b. No new production change +or install is needed merely because the unchanged scientific498 fleet exhibits +the known failure. No scientist source, image, table or endpoint was inspected +or modified and no author was contacted. + +One targeted receiving diff of Root394 confirms that its MaterializationBatch +retains original typed Output objects through the same replace-based source +fallback/variable-component hooks and successful-save records. The narrow point +options, writer, source-identity property and artifact source-query seams remain +as previously released; Root's additional publication machinery does not yet +repair them. No generic provenance/compiler/load/stack patch is required here. +This receiving review is source evidence only, not execution against Root394. + +Remaining owner boundaries: Planck owns494 source closure/review and future +engineering point-producer acceptance. Parent owns integration/release of an +eligible ordinary target and a resource-admitted engineering endpoint, plus152's +continuous live/reopened native journey. Root retains134 broader persisted +publication. Scientific498 remains immutable. A later engineering journey must +exercise actual live point settlement and persisted reopening separately; these +are not inferred from source controls or an author's manual reopening. + +Ordinary installed packaging checkpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reviewed main e9e79ac41 was normally merged at39244bab1d47d5af3c87ff96f70014ff091c67ee; +the three reviewed point repair files and original guard remain unchanged. +The ordinary wheel and new private target are retained under +``/home/ts/wt/openhcs-issue-batch-20260929/engineering494`` (wheels01/target01). +Wheel SHA256524cb5262f7fa2f396c5018e4cce22bf0e13847d58ea212200cf43ea1b78bdf9. +The original setup hooks, existing paired interpreter and no-dependency pip +route were used, without a new environment, checkout or download. + +The original wheel verifier and current declaration-owned knowledge projection +confirm815 tracked OpenHCS/benchmark source/wheel/target byte matches,90 declared +knowledge/skill assets, all13 skill files and both native C++/ABI3 extensions. +Actual installed-owner origin assertions preceded seven passing original tiny +writer/projection controls; two pre-existing pytest configuration warnings +remain. These exercise real disk ZIP save/readback and unchanged geometric +guards, not an MCP/native viewer. Build/install/installed-control cgroup peaks +were283799552/106414080/378204160B respectively, under512MiB/noSwap/CPU1/60s; +all terminal0, swap/OOM0, scopes inactive. Five foreign gitlinks remain dirty +and untouched. Full hashes, commands, ownership and release boundaries are in +``engineering494/QUALIFICATION01.rst`` and original logs. + +Source and installed writer/projection behavior are ready. Actual public MCP +live settlement and persisted reopening/alignment remain unverified until +parent releases one exact engineering slot; no runtime/viewer was launched. + +Public journey preparation checkpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Parent named the future existing H00294 helpers (VNC5998), native6012/7012 +and viewer6013/7013; custody remains scientific and NOT released at preparation. +Complete ``pipeline-live01.py``, reviewed ``point_volume_fixture_494.py``, +global config and original journalled ``launch-live01.sh`` are now saved in +engineering494. ``LIVE01-PREPARATION.rst`` records the ordered producer, +automatic-settlement, same-viewer persisted reopening, calibration/feature +linkage and genuine XY/XZ/YZ acceptance journey. Only the original four own +5x7 uint16 source planes are admitted; source-domain/metadata construction +remains with the original declarations and runtime. No SCI source is consulted. + +Stdlib AST/JSON and shell syntax checks pass (``preparation01.log``); this is +NOT registration, public validation/compile or native acceptance. No project +imports/client/native/viewer start occurs. Runtime remains one aggregate4GiB, +noSwap/CPU1, ordinary10s observations and original handle custody. Packaged +target01 stays unchanged and requires no rebuild for this documentation-only +checkpoint. Actual slot/resource admission and every live contract are pending. +Actual public failure03 and source-owner repair04 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Parent released the original programme on recovered94. Original client +PTY39222/InvocationID1a31b8490e614741865cca14e6775244/start1790998807 ran +unchanged target01. The read-only original input could not admit workspace +initialization; four326B TIFFs were staged byte-identically under admitted +output/input-staged01, preserving original plate_path through the public +execution_plate_path contract. The original rejection and session-1 remain; +only distinct session-2 compiled and executed. No source or registration replay. + +Compile871111d1-764e-434f-a8f9-e2c22329839d COMPLETE. Single public execution +113462a1-229b-478a-b4a5-0e1cd1a53cb7 FAILED in original point writer/core3268, +ROIFractionalZ.source_component_domain92: ``3D point ROI source planes vary +outside Z.`` This occurred before returning PointROIOutput/point settlement. +One main image route streamed; no point archive or runtime-values export was +completed. Actual raw PNG opened personally. Point triples/calibration/row +alignment/reopen are MISSING, not a native acceptance. All original journals, +full trace, PNG and typed closes are frozen in engineering494/LIVE03-TERMINAL.rst +and LIVE03-FROZEN.sha256. Native2005980/create1790998932.22 and viewer2039581/ +create1790999389.18 both public ACK/process_exited/endpoint_terminated/succeeded +true; four ports and all owned PIDs gone, original scope inactive. Client exit1 +is retained (earlier strict argument rejections), separate from failed science. +Peak2403676160B/noSwap under4GiB. Helpers and three SCI remain untouched. + +Actual public compiled source facts are retained in +engineering494/PUBLIC-PLANE-FACTS03.json. The original guard's only non-Z raw-map +disagreements are OriginalSourceMetadata containing literal Z000..003 and +SourceFilterPaths containing each distinct original TIFF path. Canonical +well/site/channel/time and source calibration agree; Z is ordered0..3. Failed +runtime table metadata was not exported, so this is compiled-source evidence, +not a fabricated recovered runtime table. Original source flow passes complete +plane provenance through the measurement projection to the original writer. + +This is IDEN-7 (a check wider than the coordinate question) and BOUND-2 (bypassing +the existing component owner). Fix ONLY ROIFractionalZ.source_component_domain: +derive coordinates through source_component_metadata_items and its original +SourceComponentProjectionStrategy/SourceMetadataRoleView, compare nominal +non-Z membership/values, validate uniform SourceVoxelSpacing through its owner, +and return the FULL original domain unchanged. Delete raw all-key equality; +do not discard provenance, duplicate a component roster, fabricate canonical +metadata, modify compiler/source construction/stack, or weaken Z bounds. +Original writer and persisted-reopen service already consume this one guard. +Necessary exact seam was posted to Root394, comment5965305655. Parent explicitly +released source-only494 repair; no new runtime/wheel release is inferred. + +Original refactor-audit AST traversal was reused file-at-a-time, including +registered coordinate/role/calibration families and every guard consumer: +engineering494/AST-FAMILY04.json,704 OpenHCS and95 actual paired PolyStore/ZMQ +modules, no parse omissions. Named AST coverage is not a global NRA proof or +dynamic equivalence proof. Current main87d9a99a integrated normally; determining +#509 changes only knowledge-service/test, no point production seam. All five +foreign dirty gitlinks and immutable installed target01 are untouched. + +Published source87c841f16 independently reproduces the original rejection from +EXACT retained public plane records at unchanged target01, then accepts the +same full unmodified records at corrected source. Original reproduction +terminal0/32,124,928B/noSwap/OOM0. Source05 controls:25 PASS/6 FAIL in4.36s, +terminal1/319,324,160B/512MiB/noSwap/CPU1/60s/OOM0. Original test-plugin harness +failure04 retained; distinct05 adds only the actual test-package import root. + +Passing25 retain genuine non-Z/invalid/duplicate/disordered/fractional Z, +calibration/unit, original writer/ZIP/anchor/projection and payload guards. +Six enriched-source writer/ZIP checks preserve every original metadata value, +but full provenance identity differs after nested JSON keys reorder. No assertion +is relaxed or fixture artificially sorted. Root-owned +source_image_provenance._component_metadata_identity uses repr of nested raw +maps; existing SourceMetadataIdentityProjection already owns canonical nested +identity. Exact delegation hunk and six failures posted Root394 +comment5965364042, not patched across its protected source owner. Named remaining +dependency is that precise seam release/owner implementation. Full receipt: +engineering494/SOURCE04-QUALIFICATION.rst. No fresh wheel/runtime or installed +native acceptance is claimed; original03 stays FAILED/frozen. + +Separate domain/rejection qualification06 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Distinct retained-source check selects only point_source_admission and +point_domain_preserves:19 PASS/12 DESELECTED in4.24s, terminal0. Actual guard +origin is source04/openhcs/core/roi_point_metadata.py. MemoryMax512MiB, +peak307187712B/noSwap/CPU1/60s/OOM0; scope terminal/inactive. This proves those +admission/domain rejections only, NOT full31PASS or installed native acceptance. +Original25PASS/6FAIL and all enriched writer/ZIP identity failures remain +unchanged. Root394 bb9c has no release for comment5965364042; shared source is +not patched. Original failed public journey, logs, source and scratch preserved. +Evidence: engineering494/domain-rejection-control06.py and matching log, +DOMAIN06-GUIDE06-QUALIFICATION.rst. No build or service launched. + +Receiving Root owner integration07 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The next determining owner check found Root394 advanced to4eaacf17e5de before +any helper/import edit. Normal merge337a31c9b receives its existing +SourceMetadataFields owner unchanged. No competing production hunk authored. +Its provenance_identity_items still explicitly preserves nested insertion-order +repr; adjacent identity_items owns nested key canonicalization. Exact remaining +owner method hunk posted394 comment5965533217 for Root implementation/release. + +Named-family AST update704 OpenHCS+95 paired modules, zero parse omissions. +Actual source07 batch:27PASS/7FAIL in4.85s, terminal1, peak298295296B/512MiB, +Swap0/CPU1/60s/OOM0. Original31 remains25PASS/6FAIL unchanged; three additional +identity controls give2PASS/1FAIL: changed nested values and plane order remain +distinct, equivalent reordered mappings fail. This is NOT repaired identity, +full31PASS, wheel/native settlement or biological acceptance. All original six +failures and public journals preserved. No assertion weakening/fixture sorting, +live target changes, new runtime, build or environment. Five foreign gitlinks +untouched. Evidence engineering494/ROOT07-QUALIFICATION.rst, +IDENTITY-FAMILY07.json, source-control07.py/.log and ROOT-IDENTITY-HUNK07.rst. diff --git a/docs/validation/materialization_plumbing_384_20261001.rst b/docs/validation/materialization_plumbing_384_20261001.rst new file mode 100644 index 0000000000..8125d2b4b7 --- /dev/null +++ b/docs/validation/materialization_plumbing_384_20261001.rst @@ -0,0 +1,132 @@ +One materialization batch for saving, publication and observation +=============================================================== + +This candidate addresses the generic plumbing tracked in issue #384. It is not +an accepted end-to-end performance result. The separate light owner trace bounded +eight repeated renders at approximately 0.297 seconds and metadata finalization +at 0.896 seconds; removing repeated rendering alone cannot close the generic +runtime gap. This is a structural dependency of the broader runtime boundary work. + +Ownership and production flow +----------------------------- + +``MaterializationBatch`` owns one writer-group rendering and saving algorithm; +its primary path is a derived view. ``SavedMaterializationOutputs`` owns only +completed backend outputs. ``MaterializedRuntimeArtifact`` inherits that shared +backend projection and adds the original reduced artifact binding and observation +behavior. Backend-indexed mappings contain actual destination values, not behavior +handlers. Existing WriterSpec declarations continue to determine format-specific +writing. No new runtime ledger, context cache or identity memoization was added. + +``BackendSaver`` reports only payloads accepted by each backend after successful +save calls. Failed saves raise without returning successful outcomes. Artifact +finalization passes those exact outputs to the existing nominal OutputTarget +family. Transient publication targets own the outcomes until publication finishes; +they are excluded from target equality and hashing. Image directories and source +projections consume these outputs without rendering the mutable logical artifact +again. Rendered pixel content is not carried into worker observation transport. + +The already declared ``StepExecutionObservation`` now carries actual persistent +locations and declared export paths from finalization. Ordinary worker progress +consumes returned locations; worker result observations accumulate returned paths +before image resources are released. Plate-scoped execution uses the same actual +materialization handoff. Existing ``CompiledPlateExecutionExtras`` carries its +RuntimeExecutionObservation; ``CompiledPlateExecutionResults.runtime_observations`` +is a derived collection of worker and parent observations. Both ZMQ value and +outcome exports consume that collection. RuntimeExportObservation no longer +reconstructs output paths from compiled contexts and mutable logical values. + +Terminal materializations persist without external-export participation; +streaming-only specs report no persistent destinations. Measurement reducers can +produce synthetic records absent from RuntimeValueStore; outcomes remain bound to +the record actually reduced and materialized. Main-flow manifests retain their +independent inheritance and semantic-address contracts. + +Removed code and intentionally retained consumers +------------------------------------------------ + +Removed ``_materialization_output_groups`` after moving its shared algorithm to +MaterializationBatch.render. Removed the tests-only materialized and observed +materialized path helpers, the obsolete current-runtime export-path helper, the +single-call observed-materialization wrapper, and +RuntimeExportObservation.from_execution_contexts. Routing tests now intercept +batch preparation; physical/export tests pass actual outcome records explicitly. + +Historical debug reuse still has an explicit prediction surface named +``preview_reused_*``. It does not create a new-save StepExecutionObservation. +Snapshot-owned physical receipts remain OPEN for that separate debug lifecycle; +these prediction functions are not an ordinary execution fallback. Historical +viewer expectations retain their real declaration-based rendering consumer. +Analysis consolidation retains its real CSV-content derivation consumer; it is a +separate reduction, not a replacement authority for observed export paths. + +Evidence and limits +------------------- + +Original source: 10f7e7075b412ccea8959f2c4e1aa818455149fa. NRA original syntax/class +census covers 703 modules, 5157 original ClassDefs, 5145 canonical projections and +12 retained OPEN syntax rows. The bounded ownership decision and complete census +are retained in the external benchmark evidence directory. The authored lifecycle, +new return values and argument migration exceed declaration-movement DSL proof +scope; no native behavioral equivalence proof is asserted. + +The first consumer gate retained 154 passing and 33 failing tests. Routing spies +intercepted the removed production boundary; metadata fixtures omitted the actual +output handoff. Their assertions were preserved while moving those fixtures to the +new boundary. The next six-module gate passed all 197 tests. A later parent gate +passed 264 tests with three new worker-fixture failures: the fixture failed the +existing DebugExecutionContext nominal boundary. It was replaced with a genuine +ProcessingContext subclass; the production boundary was not weakened. + +The final eleven-module gate passes 267 tests in 5.12 seconds. It includes actual +backend acceptance and failed-save controls, exact output-object reuse after +logical input mutation, rejection of re-rendering during ordinary worker progress +and export, all three runtime observation modes, pickle transport after resource +release, exact combined worker/parent export paths, materialization formats, +metadata publication/reconciliation, runtime/viewer validation and ZMQ consumers. + +The original unmodified R0 guard passes for a264ee95546ab17a3ded40f81759cc6234d08817 +against main e341ac9b55cb2b6a143df9083785b296c23000df. There are no positive debt +deltas; ZMQExecutionServer god-class excess decreases by eight lines and the +artifact materialization foreign-absence probe decreases by one. This revision +includes the normal merge of main's fixture/documentation fix; production sources +are identical to the 267-test candidate. + +Combined R1 source qualification, representative ordinary unprofiled timing and +native numerical parity remain required before promotion. No benchmark ratios or +speedup claim are attached to this receipt. + +Actual array outputs and raster publication +------------------------------------------ + +The first combined native comparison retained 28 numerically equivalent cases +and two illumination metadata failures. The saved illumination NumPy arrays were +actual outputs, but their NumPy-only directory had incorrectly become eligible +for raster image inventory publication. The existing FileManager inventory +includes declared raster formats; NumPy array exports are outside that inventory. +The candidate had removed OutputTarget's inherited ``contains_images`` admission +gate while switching directory discovery to actual saved outputs. + +The correction restores that existing nominal gate. Actual NumPy saved outputs +and their observation locations remain intact; publication and completed-plate +reconciliation consume only directories with images in the existing inventory. +No second format registry, rerendering fallback or empty source projection was +introduced. + +Independent controls use a real FileManager and physical NumPy-only and mixed +TIFF/NumPy outputs. The original candidate fails the NumPy-only control with the +same empty SourceProjectionSet error; its mixed-directory control passes. Both +controls pass after restoration, verifying exact saved pixels, actual observation +paths, raster-only inventory and typed calibration. The focused red and green +receipts are ``materialization-npy-publication-red-v3-20261001.log/xml`` and +``materialization-npy-publication-green-20261001.log/xml`` in the external benchmark +evidence directory. Earlier import-order and typed-unit fixture errors are also +retained separately; they are harness failures, not the product regression. + +The expanded eleven-module consumer gate passes all 269 tests using only the +original nominal viewer route fixture from the root test configuration. Two +initial broader invocations retained 264 passes and five missing-fixture setup +errors; the corrected fixture registration preserves those same five controls. +The final receipt is ``materialization-npy-publication-consumers-v3-20261001.log/xml``. +Full native parity and fresh accepted timing remain required for the corrected +combined source; the original 28-pass/two-error receipt is not replaced. diff --git a/docs/validation/persisted_result_publication_134_20261002.rst b/docs/validation/persisted_result_publication_134_20261002.rst new file mode 100644 index 0000000000..fe8e061086 --- /dev/null +++ b/docs/validation/persisted_result_publication_134_20261002.rst @@ -0,0 +1,47 @@ +Saved-result publication through the existing output owners +========================================================== + +PR394 integrates the source proposal supplied by PR404 (issue134), against +shared runtime source4afd51415. Successfully saved result directories participate +in publication even when their files are not raster images. This changes the +shared publication contract, not an ROI-specific writer or a function case. + +Existing OutputTarget owns guarded contains_outputs and a stored_output_paths +hook. RuntimeArtifactMetadataTarget overrides that hook for all saved formats; +no additional capability class is needed for its sole existing consumer. +The original MaterializationBatch saved outcomes determine runtime destinations. +Publication cannot rerender a mutable batch or invent an outcome from unrelated +filesystem files. Exact plate-relative result-directory ancestry is retained. +Final reconciliation derives previously declared directories from the existing +OpenHCSMetadataHandler after runtime values have been released. + +AtomicMetadataWriter remains the single transaction. It constructs image +component projections only for nonempty addressed image sets. SourceProjectionSet +retains its nonempty invariant, and actual saved images without produced source +addresses still fail without changing the metadata document. Result-only +publication produces an empty image inventory and projection, not an invented +image or source address. Existing format inventory remains authoritative. + +The current-source original control reproduces four failures and one pass. +The actual two-owner implementation passes89 publication, checkpoint, metadata +and materialization-journey tests. Five source cases exercise real bundle writer +saving, simple and nested result-only directories, original public result queries, +reconciliation after release, byte-exact refusal of unaddressed images, and a +new independently declared target with cooperative publication hooks. Its test +declaration explicitly requests its own registry key and restores the complete +registry after use; the initially inherited-key test contamination is retained. +The existing NumPy export control now requires a declared result destination +with empty raster inventory, instead of incorrectly requiring no metadata file. + +The NRA owner inventory joins137 original declarations across the selected +existing owners, scans1,498 caller modules without parse failures and records +alternative owners and open binding leads. Those syntax facts are not proof of +runtime binding or an architecture-gate pass. Original-source failure, failed +initial test path, registry-contamination failure, corrected consumer gate and +inventory receipts remain in the shared benchmark-runs workspace under +pr394-roi-publication prefixes. + +This is saved-output admission correctness and establishes no performance gain. +Synthetic archive bytes do not establish native ROI geometry, scientific pixels, +source provenance reopening or complete issue134 graph acceptance. Those broader +PR404 gates remain open. No additional scientific pipeline is executed here. diff --git a/docs/validation/prepared_callable_signatures_419_20261002.rst b/docs/validation/prepared_callable_signatures_419_20261002.rst new file mode 100644 index 0000000000..e987dab3b9 --- /dev/null +++ b/docs/validation/prepared_callable_signatures_419_20261002.rst @@ -0,0 +1,54 @@ +Prepared callable signatures and raw argument admission +====================================================== + +Registered argument contracts belong in function-library warmup before server +readiness. Newly authored declarations prepare before compiler validation and +resolve once per compilation. Invocation must consume those prepared contracts. +This is a correctness/preparation dependency of the plumbing work, not a claim +that signature caching alone removes multiple seconds. + +CallableMetadata carries the resolved canonical inspect.Signature. Parameter +names, annotations, defaults and semantic controls derive from that declaration. +Preparation publishes it only after every registered preparation hook finishes. +FunctionReference and existing namespace/pickle/cloudpickle transport retain it. +The compiler reuses registered snapshots and captures fresh authored snapshots; +the old late invocation preparation is removed. + +Actual boundary inspection found that most CP raw targets differ from their +semantic signatures because injected controls are absent from the raw function. +The existing metadata owner therefore also carries a raw-runtime Signature only +when required by that distinct boundary. Identical boundaries reuse canonical; +default objects are retained by identity and never compared through value equality. +Existing invocation and batch-request owners receive the exact raw view for +filtering/defaults. Three independent signature/default/type LRUs and the obsolete +signature-parameter helper are removed. Unprepared authoring and distinct generic +wrapper queries remain live. Explicit library rewarm or supported declaration +invalidation refreshes registered snapshots; arbitrary direct annotation/default +mutation does not retroactively rewrite compiled contracts. + +Raw array admission is derived from the callable's declared nominal type, including +unions and Annotated. The adapter retains context until the actual raw boundary. +Array-only callables receive pixels; declared RuntimeArrayData consumers retain +metadata. Their truthful annotation migration changes no scientific function body, +defaults or decorators, as checked across all 19 affected CP modules by AST. + +Controls: 646 passed, 5 existing skips, 2 existing warnings. They cover preparation +ordering, rewarm/new compilation, request bindings, custom signatures/defaults, +transport, and 12-plane compiled execution with all signature/hint queries forbidden. +An additional 75 publication/real-payload controls pass, including actual compiled +ColorToGray spatial-mask/channel collapse, MedianFilter intensity scale/mask and +IdentifySecondaryObjects source coordinates/provenance. + +The final real isolated registry check prepares 267 catalog entries and 976 targets; +all 267 contracts have snapshots, with zero missing signatures and zero subsequent +owner signature/hint resolutions in contract queries/projections. Its 16.67s +cached-startup duration is diagnostic startup and excluded from pipeline clocks. +The earlier fresh-cache 91.01s startup check remains separately preserved. + +Evidence is retained under /home/ts/code/projects/openhcs-benchmark-runs: +issue419-shared-signature-closure-final-receipt-20261002.json, +issue419-actual-shared-signature-closure-readiness-20261002.json and +issue419-truthful-annotation-actual-source-consumers-20261002.json. Earlier failed +intermediates are retained. Current all-case scientific parity and ordinary +performance are separate pending gates; earlier 90c30 timings do not qualify this +new source or updated shared dependencies. diff --git a/docs/validation/shared-dense-coordinate-owner-20261002.rst b/docs/validation/shared-dense-coordinate-owner-20261002.rst new file mode 100644 index 0000000000..4d865af45f --- /dev/null +++ b/docs/validation/shared-dense-coordinate-owner-20261002.rst @@ -0,0 +1,111 @@ +Shared dense coordinate reduction +================================ + +The shared reducer is integrated into PR394 for issue #384. The isolated +qualification source was ``d5441f960fb4cafe5a1f56740cf8c52de7a7d69d``, normally +merged with qualified PR394 ``eb2f23efd`` and main ``a6c18f054``. Integrated +controls use clean ``6e69b85122d3c1a44857835ba9b3fe77e743240d``. + +``ObjectLabelStorageStrategy`` retains the genuine sparse-IJV algorithm and +owns shared sum allocation/division. The existing dense leaf specializes int32 +2D/3D arrays; ``ObjectLabelValueStorageStrategy`` delegates actual storage. +Tracking and relationships consume one core numerical primitive through their +existing ABI adapters. Their two duplicate kernels are deleted, with all five +compiled consumers migrated. No new class, registry, dispatch table or cache +was introduced. + +Before a public domain iterator can mutate pixels or raise, the dense leaf +captures coordinate moments from a borrowing strided 3D view. The existing +``DenseIntegerObjectLabelIdDomain`` bounds moment allocation by the number of +positive pixels. Empty or large sparse ID spans use the original sparse path; +a lone ID ``2**31 - 1`` therefore reaches the original domain error without +allocating a max-ID-sized table. Unsupported dtypes/ranks retain original +conversion, warnings and errors. Readonly, reversed, Fortran and zero-stride +inputs avoid flatten copies. This establishes an O(nnz) allocation bound, not +identical heap thresholds under OOM or a resolution of issue #433. + +Existing provider/callable preparation warms C/F/A writable and readonly +signatures before execution. An isolated PrimaryObjects preparation control +refuses both compilation and cache loading during subsequent admitted calls. +Controls preserve effectful Sequence iteration, NumPy allocation-phase errors, +NaNs/counts, independent result buffers and input mutation/alias semantics. +Eleven effect controls also run against the actual unchanged Git parent. + +Qualification +------------- + +* Isolated synchronized controls: 100 passed, 2 existing skips. +* Integrated controls at ``6e69b85122``: 101 passed, 2 existing skips in + 194.57 seconds; tracked source hashes remain unchanged. +* Original scoped R0: every metric delta zero; original R1: no increases, + unchanged 160-second budget. Both compare ``eb2f23efd`` to ``d5441f960``. +* Global NRA census: 5,526 original classes, 5,513 canonical declarations, + 13 explicitly OPEN syntax rows, no parse errors. +* Saved real Wound pair: exact coordinates/counts and physical allocation/ + alias gates; original median 0.160669 s, candidate 0.019144 s, local saving + 0.141524 s across paired replay. Preparation/JIT precede timing. +* Twenty-one saved Track frames and old-Git tracking/relationship numerical + ABI comparisons pass. This is not a full relationship transaction proof. + +Initial admission, readiness and high-ID negative evidence remains retained. +These local results alone do not establish ordinary pipeline performance. +Root-owned ordinary qualification is recorded separately below. Illumination +has no centroid calls. + +Evidence: ``/var/tmp/openhcs-shared-dense-coordinate-owner-closure-v2-20261002.json``; +``/var/tmp/openhcs-shared-dense-production-saved-replay-v4-20261002/receipt.json``; +``/var/tmp/openhcs-pr394-integrated-dense-coordinate-controls-20261002.log``. + +Ordinary integrated comparison +------------------------------ + +Four uninstrumented public sweeps in ABBA order compare frozen b4700288f +(shared geometry baseline) with integrated 6e69b8512 (shared coordinate owner). +Each case has one well, inline worker and thread on CPU5, default OUTCOMES +and memory observer, and a reused READY server. Server/library/kernel startup +and shutdown are excluded; compilation remains in total. No samples are removed. + +Two-sample descriptive means (seconds; not statistical significance): + +* ExampleWoundHealing: execution 3.287030339 -> 2.984141707; + compilation 0.337689161 -> 0.340739846; + total 3.866369809 -> 3.562642806. +* ExampleTrackObjects: execution 5.338271976 -> 5.052027345; + compilation 0.929434419 -> 1.252314091; + total 6.629491309 -> 6.684317434. + +Track total regresses by 0.054826125 seconds in this comparison. Its compilation +increases by 0.322879672 seconds; no causal late-JIT attribution is established. +Wound total decreases by 0.303727004 seconds. Neither result satisfies the +remaining whole-pipeline objective or replaces the generic-plumbing frontier. + +Every observation, in case/compile/execution/total seconds: + +* 0-A ExampleTrackObjects: 0.950359821, + 5.625208855, 6.914708113. +* 0-A ExampleWoundHealing: 0.343463182, + 3.276733875, 3.827193568. +* 1-B ExampleTrackObjects: 1.421878338, + 5.020748138, 6.829343345. +* 1-B ExampleWoundHealing: 0.342035055, + 3.044292212, 3.623388787. +* 2-B ExampleTrackObjects: 1.082749844, + 5.083306551, 6.539291524. +* 2-B ExampleWoundHealing: 0.339444637, + 2.923991203, 3.501896825. +* 3-A ExampleTrackObjects: 0.908509016, + 5.051335096, 6.344274506. +* 3-A ExampleWoundHealing: 0.331915140, + 3.297326803, 3.905546051. + +All eight complete authored-output inventories, strict table/image comparisons, +scientific file bytes and source/environment/native/input freezes pass. Wound +retains full native qualification through the exact qualified output reference. +Track preserves all original OH bytes and its original 77,726-pixel native RED; +this does not include the separate draft renderer repair in PR468. No parity +tolerance, exclusion or output policy changes. + +Receipt: ``/var/tmp/openhcs-pr394-shared-dense-ordinary-abba-science-20261002.json`` +SHA256 ``affb93c5955585be6963936552a1f40da051989688e7b0c58f406f398e894400``. +Commands: ``run_pr394_shared_dense_ordinary_abba_20261002.py`` and +``validate_pr394_shared_dense_ordinary_abba_science_20261002.py`` under /var/tmp. diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst new file mode 100644 index 0000000000..ef92751224 --- /dev/null +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -0,0 +1,2178 @@ +Shared runtime plumbing qualification +==================================== + +Issue #384 covers repeated source resolution, metadata construction and artifact +rendering outside callable execution. This branch combines three production +dependencies under their existing lifecycle owners. It does not yet establish an +end-to-end speedup. + +Primary target and abstraction admission +--------------------------------------- + +The primary performance goal is shared runtime work outside callable execution: +invocation and adapter preparation, stack loading, output registration, +finalization, saving, publication and provenance. Each existing abstraction must +own a distinct invariant, behavior or useful validated view. Objects that only +forward or reconstruct facts already owned elsewhere should be removed, with +consumers deriving directly from the determining owner. Shared algorithms belong +on existing nominal ancestors; a new class is not evidence of improved ownership. +Small measured redundancies are folded into the coherent migration even when +they cannot independently close the performance gap. + +The ordinary 3D probe at clean revision 5d2fd489946e6885f161a7379e4eec1d14e307fd +records 9.860550 seconds of public server execution. Its same-thread root takes +9.757393 seconds, including 4.097127 seconds at runtime callable boundaries and +5.118847 seconds elsewhere inside steps. Final plate publication adds 0.476819 +seconds. The callable boundary includes target resolution and signature filtering; +it is not an exact naked-kernel timer. Execution is inline for this one-worker +configuration; a configured fork policy does not imply a separate worker process. + +A deeper probe on the same source records 10.275668 seconds, including 0.023519 +seconds of explicitly isolated input capture. Its nonoverlapping coarse phases +locate 0.697612 seconds in output recording, 0.478745 in image-request preparation, +0.795071 in three atomic publication transactions and another 0.143612 in durable +target reconciliation. Remaining loading and output-registration work takes +0.739507 and 0.722780 seconds respectively. Stack metadata composition takes only +0.102824 seconds across all consumers. These diagnostic clocks locate work and +must not substitute for an ordinary uninstrumented paired comparison. + +Required output work remains included: this pipeline saves 120 TIFF planes and +six result CSVs. Some work outside the callable boundary is numerical table +assembly or actual saving, so the full noncallable duration is not asserted to be +removable indirection. Nine stack-composition calls belong to loading; nineteen +belong to unstacking. Aggregate composition counts do not establish cache misses. + +Saved real-payload replays confirm that duplicated input/name resolution and +rebuilding a compiled callable contract have small individual payoffs: about +0.112 milliseconds per input pair and 4.386 milliseconds per 211 resolutions. +These are included for ownership cleanup, without attributing the larger adapter +gap to them. The direct cache payload migration removes +``RuntimeImageStackCacheValue``, which owned only a forwarded ``stack`` field; +the cache retains path/memory identity, invalidation and execution lifetime. +All 102 related lifetime/source-projection controls pass for that migration. + +Complete saved atomic publication transactions retain actual parser, labels, +saved paths, pretransaction JSON, locking and filesystem writes. Deriving +geometry from the already decoded nominal projections reduces a 60-path +transaction from median 0.13556 to 0.07601 seconds and a 120-path transaction from +0.27224 to 0.15354 seconds, with byte-identical complete JSON. The production +implementation also reproduces both baseline byte hashes, and 57 metadata, +projection and image-output controls pass, including pruning, concurrent partial +inventory updates, fresh geometry and atomic rollback. This is stage replay +evidence, not an end-to-end speedup claim; integrated qualification remains in +progress. + +The existing CellProfiler executor now resolves the compiled raw callable once +at its validated invocation boundary and passes that target through the existing +processing and slice dispatch. The existing callable-view policy accepts that +already resolved target; no additional generic invocation type or parallel +contract/callable fields are needed. Image source-name projection consumes the +already resolved payload through the existing artifact strategy family. + +The executor does not own a second interpretation of input relations or returned +artifact contexts. ``ArtifactSpecCollection`` derives exact broadcast indices +from its ordered declarations; ``RuntimeReturnedOutputMatcher`` contextualizes +the compiled canonical return ABI. ``AlignedImageStack`` binds the complete +declared output roster to its exact slice contexts in one linear traversal. The +old CP and canonical-values helpers are deleted. The existing contract, +collection and returned value determine these answers directly. + +The deeper clean-source probe at 5d1f1a781eda851b4783b68db342b76eeef9dea9 records +10.053237 seconds of public execution, including 4.465789 seconds at the callable +boundary and 4.939001 seconds elsewhere inside steps. Loading takes 0.699904 +exclusive seconds, image recording 0.519127, output identity 0.412492 and image +request preparation 0.388981. Explicit diagnostic capture costs 0.044853 seconds. +These partitions are diagnostic evidence, not accepted A/B measurements. + +All 23 produced-stack cache hits correspond to previously stored output paths. +The nine misses are original inputs; seven repeat earlier source reads and cost +0.5149 seconds inclusively. Those inputs are already in memory. Caching their +mutable arrays by path alone is rejected: storage has no mutation revision and +loaded source semantics depend on binding, aliases and plane contracts. + +The next shared route is metadata ownership, rather than a second pixel cache. +Changing spatial context, intensity or image names repeatedly constructs source +provenance and copies/fingerprints its component mapping. Source loading and CP +image recording consume this common path. The saved typed ImageMath fixture +retains actual nested mapping ownership for a cold-promotion and complete-workflow +replay. An isolated query-cache payoff does not admit this migration. CP-recorded +outputs already bypass generic postprocessing; removing that hypothetical second +contextualization would not affect the measured recorder. + +The unmodified-main light trace located about 3.44 seconds in loading, unstacking, +saving and finalization for the representative 3D pipeline. CP request and output +context construction account for another approximately 1.216 seconds. These are +diagnostic bounds, not additive nested timer totals or accepted benchmark clocks. +Repeated artifact rendering alone is bounded at about 0.297 seconds; saved plane +projection replay estimates about 0.20 seconds for that individual component. +Neither is admitted as a sufficient standalone performance route. The integrated +boundary must demonstrate a material improvement in ordinary execution. + +Production authorities +---------------------- + +``ImageMetadataProjection`` owns the common single-pass image metadata algorithm. +Source-plane and leading-axis specializations provide their declared projection +semantics through inheritance. Provenance constructors consume the actual +``SourceImageProvenance`` constructor defaults, avoiding absent-field allocation +without a second default roster. Mutable scalar payload metadata retains its +independent ownership and mutation controls. + +``SourceMetadataRecord`` owns declared/parser/rule resolution. Its declared leaf +retains live input semantics, while its resolved leaf enforces owned immutable +metadata at construction. ``RuntimeSourceResolutionSnapshot`` belongs to the +existing context-local runtime source cache and retains its determining inputs. +Only the explicit execution factories take this boundary; direct constructors and +unknown paths preserve live resolution. Derived caches are discarded on process +transport. The original adapter provenance construction remains authoritative: +deep-freezing selector records must not change repr-based image identities. + +``MaterializationBatch`` owns rendering and saving. Actual saved outputs flow into +the existing metadata target family, step observation, worker observation and +plate observation. Export paths derive from these outcomes after payload resource +release. Removed rerender helpers and the old context-derived export path API have +no remaining consumers. Historical debug reuse retains explicitly named previews; +these do not claim new successful saves. The component receipt describes the +retained CSV consolidation and independent viewer expectation consumers. + +Qualification and disposition +---------------------------- + +The integrated source passed 1,362 behavioral tests. Each of the three components +passed the original unmodified R0 guard. Final combined R0/R1 passed against main +91dc52d4040890e1b88494acba7368196f6cd1e8 at tested source +dc1a371b06079ff249c8c6432953d21e0c05917b, without increasing debt allowances or +source checks. The main synchronization passed 85 help/output/source-resolution +controls. These source ratchets are scoped guards, not a formal proof of every +dynamic binding. + +The subsequent ownership/publication follow-up normally merges main +76a2d392056f4fa6cc74bc122a552edc32f24239 at candidate +84057fbefd0ad5469c766051e19b34f42e851102, including the declaration-owned +classification and singleton image-selection repairs. All 955 controls across +26 changed-path and relevant main-owner test files pass, with two existing +watershed warnings. Original unmodified R0 and R1 pass against that main, with +no metric budgets or exclusions changed. The R1 scan uses the current NRA +checkout's virtual environment and its original 120-second deadline. Earlier +failed owner-growth/context-inspection guards and the wrong-interpreter import +failure are retained. Evidence: shared-plumbing-final-local-gates-20261001.json +and the original R0/R1 logs in the external evidence directory. + +The first complete saved-native gate exposed two illumination-correction +publication failures: actual numeric NPY saves admitted directories without any +raster images into metadata publication. The inherited raster-inventory admission +now governs directories derived from successful saves. A real FileManager NPY-only +control reproduced the failure; a mixed TIFF/NPY control preserves exact pixels, +raster inventory, alias and calibration. All 269 related controls passed. The +repaired full gate completed: all 30 cases equivalent, zero differences. The +original 28-pass/two-error gate is retained. + +Repaired ordinary ABBA evidence +------------------------------ + +Two main and two candidate observations per case were retained, without excluding +samples. Revisions are the exact dc1a371b/91dc52d4 sources above. In seconds: + +========================= ======================= ======================= +Case Mean execution Mean pipeline total +========================= ======================= ======================= +3D monolayer 10.278553 -> 9.657654 12.241441 -> 11.607624 +Imaging Flow Cytometry 15.506204 -> 15.423871 17.719883 -> 17.949389 +Advanced segmentation 8.658128 -> 8.887496 11.158074 -> 11.397346 +========================= ======================= ======================= + +3D execution improves 6.04%, while advanced execution regresses 2.65%. The default +memory observer's mean peak process-tree RSS changes by +14.65% for 3D, -3.41% for +Imaging Flow Cytometry and +10.87% for advanced segmentation. Process-tree RSS can +count shared fork pages repeatedly; these figures do not prove distinct physical +allocation. Candidate memory varies substantially between the two observations. +This is mixed evidence, not an accepted broad optimization. PR394 remains draft. + +The pre-publication-repair eight-sweep observations remain retained separately; +their larger improvement is not substituted for the repaired source's results. +External evidence: shared-runtime-plumbing-repaired-qualified-abba-summary- +20261001.json and every corresponding source/input/observation receipt in the +openhcs-benchmark-runs evidence directory. + +Existing-owner follow-up ordinary ABBA +------------------------------------- + +The synchronized follow-up compares main 76a2d392 against candidate d9c9f8fe4. +The candidate's production source is the locally qualified 84057fb revision; +subsequent changes only retain the qualification receipt. Two observations per +side use the same ordinary public path and retain every sample. In seconds: + +========================= ======================= ======================= +Case Mean execution Mean pipeline total +========================= ======================= ======================= +3D monolayer 11.173033 -> 9.979570 13.166303 -> 11.915196 +Imaging Flow Cytometry 16.847054 -> 18.891492 19.058024 -> 21.417949 +Advanced segmentation 8.802229 -> 8.616838 11.271471 -> 11.113117 +========================= ======================= ======================= + +3D execution improves 10.68% and advanced execution 2.11%, while Imaging Flow +Cytometry regresses 12.14%. Its candidate observations are 17.371243 and 20.411741 +seconds, compared with 17.554674 and 16.139433 for main. Compilation is also +inflated in the slower candidate sweep. These samples establish mixed evidence, +not a statistical significance or causal attribution; PR394 remains draft. + +All 24 ordered Imaging Flow Cytometry step occurrences align. Most of the extra +time is inside steps, distributed across the slower candidate sweep. Between-step +gaps remain about 0.017-0.023 seconds; execution outside the complete step span +does not explain the regression. Existing output comparisons and shared metadata +ownership investigation continue before promotion. Evidence: +shared-plumbing-owner-followup-qualified-abba-20261001.json and its summary, +source receipts and all per-step/progress observations. + +Across all six run pairs, Imaging Flow Cytometry's two header rows and all +1,800-by-527 measurement cells match exactly. All six 3D CSV schemas, rows and +cells match, as do all 120 TIFF arrays, shapes and dtypes. Ordinary output +inventories match. Outcome-wire differences are fresh execution UUIDs and actual +run-root path prefixes; no scientific values are present in those outcome views. +Advanced's database/properties were not numerically compared, so absence of +ordinary CSV/TIFF output does not establish its parity. Evidence: +shared-plumbing-owner-followup-abba-step-output-audit-20261001.json. + +The next normal synchronization integrates main +0d07909b07870557a04f125452dc6944db77cb6c, including PR358's startup feedback and +worker-budget repair, at candidate fdf50241aa9adc6d96bc9c530dcc9acd19692b64. +The recorded ZMQRuntime dependency 668edafcb5ee530a163377c936431ae1e8334e85 is +checked out in both trees and consumed through the shared editable environment. +All 498 preparation/startup/CP integration controls pass, with six skips and two +existing warnings; original R0/R1 pass with unchanged budgets and exclusions. +Issue355 is closed by its existing merged PR. The ABBA remains pinned to its +measured sources; no timing result is projected through this synchronization. +Evidence: shared-plumbing-main355-local-gates-20261001.json. + +Subsequent normal synchronization consumes documentation/knowledge changes in +main PR405 and PR406, through e66500e7ae3804abca1396681c50e1cef5e9a11e. These merges +do not change production source or dependencies. All 23 knowledge-transfer controls +pass on the final merged declarations, with two existing warnings. Benchmark and source-gate +observations remain pinned above. + +Independent review of the isolated metadata-owner migration found the important +distinction between ordered selection-record equality and public mapping-content +equality. It also checked duplicate stringified keys, normalization error order, +output-extension insertion order and fields-only transport. That migration remains +outside this PR until the affected consumers preserve their existing semantics +and the complete production-owner replay and source gates pass. Review evidence: +source-metadata-owner-independent-review-20261001.json. No stage replay is promoted +as an end-to-end gain. + +Fresh native qualification and next dominant route +------------------------------------------------ + +The revision-pinned matched Imaging Flow Cytometry driver completed three observed +native CP/OpenHCS repetitions and one warmup. All observed repetitions have zero +output differences under the existing numerical tolerances, exact discrete/schema +checks and strict declared output inventory. The input is one well with two native +image sets, one OpenHCS worker and one native job; the configured worker start +method is fork. Native and candidate phase boundaries are retained separately. +The driver's timing_claim explicitly remains none until first-module/first-axis +boundary equivalence is qualified. No native throughput ratio is asserted here; +the physical 3D volume/plane consumer gap also remains unfinished. + +Fresh paired diagnostic traces cover all three workloads with identical owner +hooks. Their high-frequency instrumentation perturbs costs, so these are authority +and call-count evidence, not accepted timing clocks. The 3D trace reduces +SourceImageIdentity construction from 142,412 to 78,769, but still calls +SourceMetadataRoleView.scalar_items 194,349 times and source metadata field identity +1,120,064 times. Image-processing invocation remains approximately unchanged in +these traces. Repeated metadata queries are the next potentially material generic +route; saved representative replay must establish its transferable payoff before +another production authority is admitted. Immutable query ownership must retain +live public mapping behavior, lazy validation, alias/role semantics and image +identity fingerprints. + +Source snapshots structurally retain historical projection owners, but output +publication does not normally replace these workspaces' cached metadata documents. +The observed RSS increase is therefore not attributed to snapshot generations. +Rendered outputs also remain visible through immediate publication; many arrays +already belong to runtime storage. Per-process memory and ownership diagnostics +must distinguish extra references from newly allocated buffers before changing +that boundary. + +The subsequent normal main396 merge changes documentation, knowledge resources +and five knowledge-transfer test assertions. OpenHCS production source bytes are +unchanged from the corresponding measured candidate and baseline revisions; the +existing source/benchmark evidence remains pinned to its original revisions. + +Issue and branch audit +---------------------- + +The source components are integrated in actual draft PR394, formally linked to +issue384. A branch/history audit covered 46 local worktrees and 194 remote branches; +the two subsequently created branches have PR396 (merged) and PR397 (draft). +Current branch tips were checked beyond historical PR association, including exact +source patch equivalence for squash/rebase history. No additional useful +unpublished fix branch was identified. Rejected prototypes and obsolete planning +branches are not nominated as unfinished production fixes. + +The subsequent complete audit verifies eight of 26 open issues have formal +closing PR references and exact manual ConnectedEvent entries; 18 still lack an +actual complete fixing PR. Mere cross-references and diagnosis-only drafts are +not counted. The pairs are 138/207, 213/217, 320/217, 355/358, 384/394, 386/388, +395/397 and 398/399. For the three automatically detected links whose manual +mutation was initially a no-op, the exact original PR bodies were restored after +the manual links were created. Head, base and draft state were preserved. +Audit evidence: official-development-link-final-20261001.json/.md, +manual-development-relink-exact-events-v3-20261001.json, +remaining-remote-branch-pr-audit-20261001.json and local-worktree-branch-inventory- +20261001.json in the external evidence directory. Counts describe that pinned +audit rather than future repository state. + +The earlier runtime-plumbing prototype 2253ec63b is rejected. Its paired timing +did not establish a gain and its original R0 gate failed. It is not part of this +branch or the accepted evidence. Failed controls and diagnostic observations are +retained in the external benchmark evidence directory rather than replaced with +the fastest observations. + +Pipeline clocks exclude ZMQ server startup, mandatory registry/kernel prewarming +and shutdown. Timed runs use the ordinary public OUTCOMES path and default memory +observer, without profiling injection. Candidate and baseline must have frozen +clean source revisions and the same dependencies, affinity and environment. + +Global lifetime counterfactuals: rejected routes +---------------------------------------------- + +The next ordinary diagnostics and exact saved-input replays reject several +insufficient standalone routes before further implementation. These are payoff +bounds, not candidate speedups. Their observations remain revision-pinned; they +do not supersede the mixed ordinary ABBA above. + +* The eight complete saved IFC geometry owners take 0.4745 seconds in aggregate + median replay; the actual complete spreadsheet renderer takes 1.3662 seconds. + Even eliminating both separate phases has an optimistic ceiling below two + seconds. The isolated geometry/export route is stopped. +* On main 18317499be069fbe58387137f326ac7b6ff57988, all 634 original Advanced + measurement queries take 0.6422 seconds, while complete CPA collection takes + 1.1943 seconds. Fabricated missing-cell counts alone do not admit this route: + eliminating both whole phases still cannot supply two seconds. The saved + query capture is a bounded subset, not a substitute for the complete clock. +* On that same main revision, all 32 3D stack loads and all saves take 0.9927 + and 0.9526 exclusive diagnostic seconds respectively. Eighteen loads select + an actual previously stored named producer; all eighteen allocate independent + main-flow pixel buffers. The full saved step-14 Resize input matches the + stored Monolayer pixels exactly, but differs in filename-derived extension + and per-plane source-name metadata. Direct value reuse would violate existing + buffer isolation and provenance. A narrow named-handoff migration is stopped. +* A low-overhead aggregate diagnostic on that revision measures all 40,673 + ``ImagePayloadMetadata.__post_init__`` calls at 0.8919 inclusive seconds and + all 147,354 ``SourceImageIdentity.__post_init__`` calls at 0.5544 inclusive + seconds. These totals overlap and must never be added. Constructor count + contraction alone cannot justify the multi-second target, even though counts + identify repeated work. The proposed 44.6 percent birth reduction is not a + measured speedup. The wider field-derivation lifetime remains unqualified. + +Every diagnostic uses the ordinary OUTCOMES path, one worker/thread and the +default memory observer. Hook overhead is retained; server startup, prewarming, +compilation and shutdown are outside execution. Exact original-method timing +does not establish that the whole measured duration can be removed. Controllers, +source/dependency freezes, successful outputs and failed earlier captures are +retained under ``/var/tmp`` and the external benchmark evidence directory. + +Evidence: ``measurement-lifetime-actual-input-baseline-replay-20261001/receipt.json``, +``global-plumbing-three-workload-coarse-lifetime-20261001.json``, +``runtime-value-handoff-step14-fixture-metadata-inspection-20261001.json``, and +``metadata-initialization-aggregate-decision-20261001.json``. The full query and +handoff observations reside in their corresponding ``/var/tmp`` captures. + +Main 9a04107492ad90233394cf17524d0ca8e74062bb is normally integrated, including +the merged native-module source-discovery repair in PR416 and source projection +capability repair in PR418. Earlier behavior/source/performance gates retain +their exact source pins. PR394 remains a draft formally closing issue384. + +The next bounded union-depth diagnostic on that main measures 3.6594 seconds +across selected metadata construction, provenance projection, component/literal +field lookup and merges. Its 972,354 selected calls enter 102,445 outer selected +regions; nested work is timed once and the final depth is zero. It retains hook +overhead, includes metadata inside callable wrappers and excludes physical +processing, I/O, complete writer preparation and publication ancestors. Thus +this is a plausible wider counterfactual scope, not 3.6594 recoverable seconds. +Reflection controls compare original and wrapped constructors: hints and errors +agree, including the original generated image-metadata constructor's existing +``NameError('InitVar')``. Actual provenance mapping decode succeeds through the +hooks. The next gate is an ordinary production comparison against the existing +nominal immutable-field owner, before further implementation or broad gates. +Evidence: ``metadata-lifetime-union-production-counterfactual-admission-20261001.json`` +and ``metadata-lifetime-union-reflection-pair-control-20261001.json``. + +Initial ordinary complete-field-owner counterfactual +--------------------------------------------------- + +The existing immutable field-owner migration is now included in this draft, +rather than left on an unpublished independent branch. ``SourceMetadataFields`` +owns the common algorithms; live declared fields and owned resolved/durable +fields provide their actual lifetime through inheritance. Owned builtin values +permit local derived-view reuse, indexed lookup and readonly snapshot reuse. +Raw mutable mappings retain fresh derivation. The obsolete role/identity view +types and duplicate field algorithms are removed. Fields-only transport drops +process-local derived caches. + +One initial ordinary, uninstrumented 3D pair compares main +9a04107492ad90233394cf17524d0ca8e74062bb against the complete candidate +401cd69fa718d6b19ce4dc35fb5f7c2d70f2d219: + +=================== ============ ============ +Metric Main Candidate +=================== ============ ============ +Execution 11.6153 s 8.1170 s +Compilation 2.2212 s 50.7997 s +Pipeline total 14.5439 s 59.6670 s +=================== ============ ============ + +Execution improves by 3.4984 seconds in this initial pair; total time regresses +by 45.1231 seconds. All six actual result CSVs have exact rows and all 120 TIFF +arrays have exact dtype, shape and pixels, with equal complete inventories. +This is an initial counterfactual, not accepted ABBA or all-case native parity. + +The compilation regression is not startup time: 239 Numba cache files are written +after axis compilation, over 48.2219 seconds, ending immediately before compilation +completes. The cold candidate source-path kernel cache exposes a startup regression. +Commit 2b86bfbe49f36d302dcd67c5b0a0cb4f656f1eb0 implemented registry kernel warming; +ffba8426cefcaa87b6b00ff056e4cb96855233e4 in PR358 subsequently removed that loop +and deferred readiness to selected compilation. PR420 restores the existing registry warmup lifecycle and closes issue162. +Its separate empty-cache regression gate is recorded below; the integrated +field-owner performance still requires requalification. + +The field-owner integration into this draft passes 81 scoped lifetime/ownership +controls with metaclass-registry 0.2.2. A subsequent projection correctness repair +passes 173 scoped controls in its isolated source. It restores source-plane +derivation and cross-invalid field/error ordering before leading-axis guards, +shares normalization on the existing metadata owner, and deletes premature +projection properties. That repair requires performance requalification; the +initial pair is not projected through it. Arbitrary external constructor/mapping +callback equivalence remains an OPEN obligation, not a claimed proof. + +Evidence: ``metadata-owner-ordinary-pair-output-parity-20261001.json``, +``metadata-owner-cold-compilation-regression-cause-20261001.json``, +``source-projection-order-384-correctness-patch-20261002.json``, +``/var/tmp/openhcs-metadata-owner-ordinary-pair-20261001/observations.json`` and +``/var/tmp/openhcs-integrated-metadata-owner-scoped-gates-20261002.log``. +Main 25d56ae3fb9b80acda80f3cf4e1c8667939147eb is normally merged; the shared +environment and PR submodule consume the declared published metaclass-registry +0.2.2 source 393a7e03003cdc56df9013f932ed4f26e632d77a. Earlier observations retain +their measured source/dependency pins. + +Restored registry readiness on main +---------------------------------- + +PR420 merges the startup correction to main +107498cfacf3b12d926c55f84c4b3cb7c04aa50f. That main is normally merged into +this performance branch. Existing affinity admission, cancellation ownership +and compiler guards for declarations introduced after startup are retained. + +An ordinary, uninstrumented 3D 1w_1t run at fix source +e55e2ec0bc98ad750837cd4dcfd830ad04096286 starts with a new empty Numba cache. +Compilation is 1.720214 s, execution 11.151797 s and pipeline total 13.637182 s. +All 481 cache files are written before pipeline submission; the last write is +6.001317 s before run creation and no cache files are written during compilation +or execution. Server startup, warmup and shutdown are excluded from pipeline +clocks. All six CSVs and 120 TIFF arrays exactly match retained main9a output, +including dtype, shape and matching inventories. This is a lifecycle regression +gate, not an ABBA performance comparison or current integrated-candidate timing. + +The fix passes 37 scoped tests with five single-affinity skips and both actual +two-core fork controls. Original unmodified R0 passes for openhcs, scripts and +benchmark; R1 reports no increases against main25d56ae under its unchanged +160-second budget. The failed Python3.12 ratchet launch and uninitialized +worktree R1 launch remain retained; final guards use Python3.14 and the initialized +main repository, respectively, without modifying either detector. + +Evidence: ``registry-startup-readiness-162-empty-cache-pipeline-20261002.json``, +``registry-startup-readiness-162-validation-20261001.json`` and +``registry-startup-readiness-162-original-r0-py314-20261002.log`` plus +``registry-startup-readiness-162-original-r1-initialized-root-20261002.log``. +The original 50.800-second compilation regression remains retained. Current +combined performance, IFC behavior and source guards remain draft gates. + +Complete-field ordinary ABBA and current ownership repairs +--------------------------------------------------------- + +The completed uninstrumented ABBA at main 107498cf against candidate 02609e12 +has two observations per source for each workload. Execution/total means are +11.162969/13.119510 s versus 8.295599/10.351318 s for 3D, +8.706567/11.262818 s versus 8.474416/11.132766 s for Advanced, and +16.500067/18.794964 s versus 16.920803/19.454312 s for IFC. Thus 3D saves +2.867370 s execution and 2.768192 s total, while IFC regresses 0.420735 s +execution and 0.659348 s total in these samples. No statistical significance +or noise explanation is asserted. All observations remain retained. + +Original unmodified R0/R1 rejected 02609e12. The actual metadata owner now owns +normalization and leading-axis transformation; the redundant raw projected +field dictionary, provenance normalization forwarder and foreign optional-state +handling are removed. Source and axis projection roles compose through +inheritance while preserving effect/error order and independent result ownership. +Common scalar admission is shared on SourceMetadataFields. The optional +intensity-proof projection and scale queries consume their existing authorities +instead of duplicating algorithms. Component controls include 41 scalar tests, +1,212 original-source behavior comparisons and 197 metadata/proof controls. +The collected classification projection census was also repaired to skip actually +abstract parameter declarations while still checking concrete descendants; +429 focused and 1,111 broader consumer controls passed before dependency update. +The inherited census failure was reproduced on main under the same imports. + +The subsequent ordinary ABBA measures frozen production candidate +90c30e1adfacd25a834ed48949b430099c866134 against the same main 107498cf: + +==================== ======================== ======================== +Workload Execution main/candidate Total main/candidate +==================== ======================== ======================== +3D 11.089386 / 8.172661 s 13.063124 / 10.106094 s +Advanced 8.854658 / 8.988559 s 11.408556 / 11.590607 s +IFC 17.382926 / 16.212604 s 19.628127 / 18.661190 s +==================== ======================== ======================== + +3D saves 2.916725 s execution and 2.957031 s total; compilation means are +1.214262 / 1.209572 s. Advanced regresses 0.133900 s execution and 0.182051 s +total. IFC improves 1.170322 s execution and 0.966938 s total in this batch. +These means do not erase the adverse026 IFC evidence or establish significance. +All twelve case observations succeed through ordinary OUTCOMES, default memory +observation, one well/thread and CPU5. Both sources use identical public drivers, +shared dependencies/native binaries and mandatory registry preparation before +readiness. Server startup, prewarming and shutdown are excluded. There are no +profiling hooks, captures or scientific substitutions in either ABBA. + +For each ABBA, all six output pairs pass scientific comparison. All six 3D +CSV exports and all 120 TIFF arrays match exactly, including complete scientific +inventories, dtype and shape. IFC has exact headers and all 1,800 x 527 result cells. +Advanced uses the existing CellProfiler database/export comparator at absolute +and relative 1e-6, with exact schemas, cardinalities, discrete values, identifier +relationships and categorical data. Eight scientific subject tables and all +relationships are nonempty. Unsaved final segmentation masks were not observed. +The90 comparison reads actualaf45 source against its still unchanged pre-update +dependency environment, attested before and after. Output files remain unchanged. + +At 90c30, original R1 passes with no increases under the unchanged 160-second +budget, and original R0 scripts/benchmark pass. R0 openhcs remains RED solely +because the scalar/container rejection classifier moved from virtual workspace +decoding to its actual field owner: source_metadata gains one local switch and +three arms, while virtual_workspace_metadata loses one switch and four arms. +The original per-file gate is not waived or declared passed. Across all changed +roots the global delta is zero switches and minus one arm; untouched files cancel +because these original metrics are file-local AST measurements. Existing error +and reported-class observation ordering is preserved rather than altered to +satisfy the metric. The global 702-module authority audit found no existing +replacement retaining that complete grammar/error contract. These are distinct +architecture and behavioral gates, not interchangeable claims. + +After both ABBA and all their comparisons finished, main +c32447f1c86a1878a313d1643a398e30ac20f75e was normally merged into this branch. +The shared environment now installs declared published ArrayBridge 0.3.6 at +source 1e53d03d9f468322a1c085c8be29485fb139caf2, openhcs-basicpy 1.3.1 and +JAX/jaxlib 0.9.2. Metaclass-registry 0.2.2 source 393a7e0, NumPy/SciPy/Numba +and both native extension hashes are unchanged. Pip check reports no broken +requirements and actual package imports pass. The separate native CellProfiler +4.2.8.1 environment remains unchanged. Earlier timings retain their measured +source/dependency pins; they are not projected through this new main/environment. +New combined qualification and full native parity remain required. + +A capture-free coarse diagnostic on frozen 80d92 source partitions one 9.208835 s +job into 4.164762 s inside RuntimeCallableInvocation.call and 5.044073 s outside. +The callable boundary includes decorated processing/metadata and is not a pure +image-kernel measurement. Diagnostic wrappers are not accepted performance +comparisons. Full load, unstack, save, CP image recording and image request phases +have a nonoverlapping 3.009640 s upper bound. Recovering 2 s requires eliminating +at least 66.453% of those complete phases, not just a cached recomposition or +an original-source cache miss. The existing named-value to eager-plane metadata +to MemoryVFS to whole-stack round trip is the next structural premise. Mandatory +named/mainflow pixel-copy isolation and concurrent durable publication remain +constraints. No new production route is admitted without a representative replay +showing that complete consumer closure can realize a material payoff. + +Retained evidence is in the shared benchmark-runs directory: +``owned-metadata-90c30-ordinary-abba-csv-tiff-parity-20261002.json``, +``advanced-ordinary90-six-pair-sqlite-parity-20261002.json``, +``pr394-integrated-90c30-original-source-guards-20261002.json``, +``scalar-owner-original-r0-global-delta-counterevidence-20261002.json``, +``integrated90-classification-concrete-projection-census-fix-20261002.json``, +``shared-main-dependency-update-c32447-20261002.json`` and +``pr394-90c30-whole-mainflow-value-frontier-20261002.json``. +Source freezes, complete observations and diagnostic boundaries are retained in +``/var/tmp/openhcs-owned-metadata-90c30-ordinary-abba-20261002/`` and +``/var/tmp/openhcs-owned-metadata-phase-probe-20261002/``. The original 026 +comparison and original failures remain separate pinned evidence. + + +Prepared callable owner and independent plane follow-up (2026-10-02) +------------------------------------------------------------------ + +Registry preparation now resolves canonical and actual raw signatures after all +function preparation hooks and before READY. Authored declarations resolve in +compilation. Immutable signature pairs travel on the existing CallableMetadata +through existing function-reference/worker transport. Runtime filtering, batch +defaults and canonical argument admission consume the associated prepared view; +three separate signature/default/type LRU caches and late kernel preparation are +removed. Server preparation remains outside pipeline clocks. + +The isolated owner repair ``760e9ae8613d8737411a627b6abfe6a15f4ca0b9`` moves actual +target resolution, signature state and carrier grammar onto CallableMetadata; +CallableProjection/Reader own preparation admission, RuntimeCallablePolicy owns +invocation, and Pure2DSliceBatchExecutor owns the formerly duplicated executor +selection. Distinct actual targets always receive independent raw snapshots; +only the same actual target shares its canonical snapshot. No new wrapper class +or equality comparison over arbitrary callable defaults is introduced. + +Original, unchanged R0 passes against both the prepared-callable commit and the +frozen qualification source, eliminating its new class-size, Boolean-chain and +foreign-absence debt. Original R1 passes within the unchanged 160-second budget +with 3,084 projections. These are isolated gates; the inherited scalar-owner +per-file R0 gate on the whole branch remains open. Focused owner tests pass 105 +controls; the larger 648-pass/five-skip suite initially retained two baseline-red +plane-domain controls. Their failure also reproduces on untouched ``94915``. + +The independent repair ``0b7f6f98a4b19d85d0dbb20f4db49e45cd0d90c1`` validates an +explicit object projection against its declared source axis and nonempty +acquisition cardinality directly. Independently authored runtime planes can own +an exact projection without acquisition-plane provenance. Physical source/label +cardinality and spatial-shape checks remain mandatory. Thirty-nine controls pass, +including both axes at one/two planes and wrong-axis refusal. No object-domain +semantics are inferred from source storage axes. + +The all-30 scientific qualification on frozen ``94915`` is RED: 24 cases pass; +four fail at missing payload context in Align/Crop, one loses its server to a +confirmed kernel OOM kill, and one differs in threshold entropy. Saved input +replay traces that entropy mismatch to missing Threshold source scale/dtype at +the canonical raw boundary. Truthful declarations are being corrected without +scientific-body or tolerance changes. Linux killed endpoint PID 2412356 with +13,360,004 KiB anonymous RSS; the dominant allocation/retaining owner is still +unmeasured. Issue #433 tracks that distinct execution-server acceptance gate and +is formally linked to this draft PR. Retained native clocks are not current +paired speedup evidence. + +Evidence stays in the external benchmark RUNS directory, including +``pr394-94915-all30-native-science-qualification-red-20261002.json``, +``pr394-signature-owner-760-source-inventory-20261002.json``, the original R0/R1 +receipts, and ``issue419-explicit-object-plane-cardinality-followup-20261002.json``. +Two partial cohort captures complete all 31 production steps and match all six +CSV files and 120 TIFF dtype/shape/pixel arrays exactly, but each admits only six +of ten graph snapshots. V4 identifies an unsupported native allocation owner in +the four remaining before-state graphs; those graphs remain missing. Diagnostic +capture clocks do not establish a speedup, and the additional multi-second +whole-value plumbing payoff remains unmeasured. + + +Prepared payload contract qualification (2026-10-02) +-------------------------------------------------- + +The integrated library/kernel readiness run at ``1ad3898`` prepares all 267 +functions and 976 targets before READY. It creates 265 distinct actual raw +signature snapshots and reuses the canonical snapshot for two identical actual +targets. Three rounds of prepared annotation, carrier, raw-signature, batch +default and invocation-construction queries perform zero additional live +signature/type-hint resolutions. The fresh-cache 90.381-second preparation is +server startup diagnostics, excluded from pipeline clocks. Later explicit-plane +validation and test-producer changes do not change these warming/signature owner +sources. This finite check does not claim zero reflection for arbitrary +unprepared authoring or unsupported callable mutation. + +The annotation-only follow-up ``9ae4c9426`` corrects twelve existing context +consumers, three forwarding helpers and CropRequest.image. Existing +RuntimeArrayData preserves their nominal image payload at the actual raw +boundary; NumPy-only consumers still receive NumPy arrays. Scientific bodies, +defaults, decorators and return declarations are unchanged by AST comparison. +Frozen ``7241ca99b`` passes native-science comparison for Colocalization, +Neighbors, YeastColonies and pixel classification with zero differences. +YeastPatches passes its previous Crop failure but subsequently exposes an +incorrect RGB source domain from IdentifyObjectsInGrid. + +Saved actual RGB pixels, source metadata and both 93-ID label artifacts isolate +the remaining defect: IdentifyObjectsInGrid's primary argument and its existing +request field/from_runtime argument falsely declare np.ndarray despite passing +the image to SourceImageObjectLabelBuildRequest. Their raw ABI strips the +1200-by-1600 source domain and (150, 170) crop origin, producing a (1255, 3) +domain for (835, 1255) labels. ``84e99c0dd`` changes only those declarations and +their direct RuntimeArrayData import. Scientific bodies and the refusing domain +guard are unchanged. Seventy-seven scoped controls and 127 integrated controls +pass. A fresh YeastPatches execution at that revision passes the existing native +comparator with zero differences. Its replay uses retained source-pinned native +artifacts, not newly timed native execution. The original 84-entry/38-module AST +frontier now has no remaining exact classmethod context leads; aliases, inherited +dispatch and dynamic routes remain outside that finite inventory. + +The five affected cases have passed across these two targeted sources. The full +all-30 latest-head science gate, the execution-server OOM retaining-owner gate, +the whole-branch scalar per-file architecture gate and fresh ordinary performance +qualification remain open. No current native speedup is inferred from retained +clocks or instrumented captures. Evidence includes +``issue419-actual-integrated-1ad-signature-owner-readiness-20261002.json``, +``pr394-7241-context5-science-qualification-20261002.json``, +``issue419-grid-context-fix-20261002.json`` and +``pr394-84e99-grid-science-qualification-20261002.json`` in the benchmark RUNS +directory. All original failed source-pinned runs remain retained. + +Graph ROI source-bearing writer integration (issue #134) +------------------------------------------------------- + +The receiving owner requested narrow integration from PR #404 checkpoint +``f3e566485a391a270a6d2c40dbfddf07881dcd3d``. The exact patch binds existing +ROIArchiveSourceMetadata to graph ROI content and places the same existing +ImagePayloadMetadata on Output. Graph provenance already selects its declared +source plane; no second global-plane selection is performed. Coordinate spacing +uses existing SourceVoxelSpacing. The binder, disk ZIP transport, geometry +projection and native admission retain their existing owners and guards. + +On current ``38067a4a5`` before the production patch, the receiving seven-case +fixture reproduces three source-bearing failures and four refusal controls +pass. After integration, all seven pass alongside the original graph, batch +outcome and materialization controls: 87 PASS in 2.28 seconds. The existing +geometry test obtains geometry through the original archive projection before +its unchanged coordinate and feature assertions; bound transport metadata is not +mistaken for a graph feature. An independent cooperative graph subtype and new +feature require no new writer or consumer registration. Missing, unbound, mixed +and conflicting native source declarations remain refused before transport. + +This is current-source synthetic writer/disk/archive qualification. Actual +installed public/native graph reopening remains with the receiving owner, and +no performance or biological acceptance claim follows. Original failures remain +in ``pr394-38067-graph-roi-before-20261002.log``; integrated controls are in +``pr394-graph-roi-integrated-after-20261002.log`` in RUNS. The historical PR #404 +branch and its older publication proposal are not merged as a whole. + +Opaque image domain and current 3D frontier (2026-10-02) +------------------------------------------------------ + +Frozen ``45f3a721e`` diagnoses the dominant current 3D failure: scalar output +composition adds a runtime axis to an opaque whole volume. Watershed receives +``(1, 60, 128, 128)`` and fails after 118.394 seconds inside its raw callable. +The failed job peaks at 3745.977 MiB. Installed Mahotas 1.4.19's filter-offset +formula requires a 2 GiB table for that incorrect 4D seed-label neighborhood; +the singleton changes neighborhood rank without changing image voxel count. +This is a source-backed bound, not an allocator-stack capture or attribution of +the original 13.36 GB OOM. No NumBa cache files are created during the observed +pipeline window. Cache warming does not correct an invalid scientific domain. + +``d092a03af`` repairs composition using the existing nominal alignment owners. +``AlignedImageStack`` retains its declared outer runtime axis; named +``ImageOutputBundle`` members supply their original individual domains. +``PatternGroupOutputData`` derives saved member declarations from those original +values, and ``ProducedOutputSemantics`` retains that domain across physical-leaf +projection and passthrough. This execution-local fact distinguishes an opaque +single-Z volume from an explicitly declared one-plane runtime stack after both +have otherwise similar leaf metadata. It does not replace durable source ingress. +Mixed cohorts cannot manufacture one combined domain. Cache misses, overwrites, +named selection and independent pixel/mask/metadata ownership are covered. +Producer and loader bundle composition use the same existing authority. + +The first scoped original R0 rejects ten added lines in the oversized runtime +class. ``6dd47dd7e`` moves actual input assembly onto existing +``PatternGroupData`` and shared independent copying onto +``ImagePayloadStackComposition``, removing the input loader's call into the +output factory. Both unchanged original scoped R0 and R1 then pass against +``45f3a721e``; 271 controls pass, including retained lazy device resolution for +bundle composition. The whole-branch scalar relocation gate remains separate +and is not waived. Cross-device composition equivalence remains unqualified. + +A fresh ordinary CPU5/one-inline-worker execution at frozen ``d092a03af`` gets +through the first 18 steps. The entire Watershed step completes in 0.156588 +seconds. The older figure is an instrumented raw-call failure, so these are not +paired successful timings or a native speedup ratio. The full pipeline still +fails at step 18, MaskImage: a correctly selected 2D invocation receives the +opaque secondary volume without a source-proved plane projection. The refusing +2D mask guard is correct. Failure-row zero compile/execute values are wrapper +placeholders; the public compile-event span is 1.619 seconds. Pipeline clocks +exclude mandatory registry/kernel server preparation and shutdown. + +The generic relation-aware repair belongs at ``RuntimeInputBindingRequest``. +MaskImage, Crop and CorrectIlluminationApply already declare +``InputStackBroadcastSourceRelation``. That declaration permits broadcasts but +does not prove spatial-Z ordering. Contributor provenance or equal array lengths +cannot supply the missing pixel-coordinate proof. ``SourceSpatialDomain`` is +currently XY-only; source/transform ownership is being traced before extending +its existing contract. No function-specific mask path or geometry-guard +relaxation is included. Latest-head all30/native parity and ordinary performance +remain open. + +Main ``902913616`` (merged PR #434) is integrated at ``ff9d7c79d``. The combined +14-file source suite has 810 passes and two unary GrayToColor failures. Both +failures reproduce unchanged on clean main ``902913616``; they are retained, +not converted into successful assertions or treated as axis-fix regressions. +No foreign PR implementation is duplicated. + +Three completed same-server Human jobs at ``45f3a721e`` pass their scientific +comparisons and leave zero backend image-array bytes after each job. This +rejects the proposed completed-job backend leak for that workload. Generic +worker resource release already executes for inline jobs. The older OOM's full +allocation mix remains unknown; no speculative global cleanup is added. + +Receipts remain in the external benchmark RUNS directory: +``issue433-rank4-watershed-large-allocation-source-bound-20261002.json``, +``issue433-three-complete-job-retention-counterevidence-20261002.json``, +``pr394-opaque-axis-input-owner-original-r0-openhcs-20261002.json``, +``pr394-opaque-axis-input-owner-original-r1-20261002.json`` and +``d092-mask-secondary-domain-caller-audit-20261002.json``. Frozen ordinary +outputs/source hashes are retained under +``/var/tmp/openhcs-opaque-axis-d092-ordinary-20261002``. The initial failed +guard-launch arguments and the original R0 growth finding are also retained. + +Persisted derived-role publication (issue #435) +---------------------------------------------- + +``ce1b7acfc`` repairs duplicate publication of one saved image occurrence in the +existing metadata writer. The existing output context derives the declared +source alias; anonymous main flow remains anonymous. Produced records and +successful materialization outcomes share a path index. A second projection is +omitted only when declared alias, artifact kind, producer scope, exact path, +scalar acquisition address and full persisted image metadata agree. Distinct +files with colliding identities, stale scopes, different kinds and conflicting +metadata retain their original refusal. No source-projection guard is relaxed. +The index is linear in produced records and successful outputs. + +The actual writer reproducer at baseline ``130c03a26`` saves two distinct role +images with real C2 addresses and calibration, then reproduces the duplicate +projection failure. At ``ce1b7acfc``, both roles publish once and reopen through +the existing virtual workspace; exact pixels, addresses and spacing pass. All +four independent refusal controls pass, together with 232 focused controls in +9.42 seconds. These are writer/materialization/reopening controls, not an +installed five-step acceptance or a performance benchmark. + +Unchanged original scoped R0 and R1 pass for ``130c03a26`` to ``ce1b7acfc``. +The first candidate ``6dfda9597`` failed original R0 for a long producer-match +Boolean and a foreign absence probe. Moving alias derivation to its existing +context owner and comparing exact producer tuples resolves those findings. +The whole-branch scalar gate remains separate. Independent review finds no +blocker; persisted occurrence identity does not prove stale in-memory pixels +equal the final overwritten file. Pixel parity remains a separate gate. + +The parent has supplied the original five-step source and bounded saved-path +facts in issue comment ``5948666249``. Actual produced-record and materialized +output pairs were not serialized there. Installed original-path acceptance +remains open; no biological replay or fabricated collision pair is claimed. +The separate #437 owner retains its singleton-consumption repair. + +Main ``8551a4864`` (merged #438 and #439) is integrated at ``3de415d63``. +The integrated publication, axis and new MCP compilation controls pass: +247 tests in 51.50 seconds, with two existing pytest configuration warnings. +The scoped guard receipts are retained externally as +``issue435-ce1-original-r0-retry-20261002.json`` and +``issue435-ce1-original-r1-20261002.json``; independent review is +``/var/tmp/openhcs-issue435-independent-publication-review-20261002.json``. +Original failures and failed guard launches remain retained. + +Fresh native comparison and measurement lifetime frontier (2026-10-02) +-------------------------------------------------------------------- + +Frozen ``919fee286f8003a00aafc023ad28767d9713842b`` completed two ordinary +public-driver observations on CPU5, one inline worker/thread, with default +OUTCOMES and the default memory observer. Server startup, mandatory registry +and kernel warmup, and shutdown are excluded from pipeline clocks. + +===================== =========== ============== ============== ============== +Case Compilation Server job Pipeline total Native CP run +===================== =========== ============== ============== ============== +Advanced segmentation 1.324832 s 8.078740 s 10.650795 s 34.327521 s +Imaging flow cytometry 1.941105 s 15.497744 s 18.177965 s 77.795280 s +===================== =========== ============== ============== ============== + +Native CellProfiler 4.2.8.1 uses the same physical input paths and SHA256 +values, CPU5 and one thread. Its invocation excludes interpreter/JVM startup, +pipeline load and one complete warmup, but includes image-set preparation, +modules, post_run and Measurements.close. These are single observations with +different explicit setup/publication accounting; they do not establish a +kernel ratio, statistical significance or a new patch's speedup. + +IFC's existing strict exported-science comparison passes with zero differences +and 1800 nonempty rows in each CSV. Native has 528 physical columns and OH 527: +the native redundant Number_Object_Number equals ObjectNumber for all 1800 +rows, and the existing nominal identity policy recognizes that declaration. +This is semantic measurement equality, not identical cross-tool CSV headers. +The observed native/OH ratios +are 5.019781 for server-job execution and 4.279647 for pipeline total. Unsaved +segmentation masks are outside these retained outputs. Advanced's existing +scientific database comparison also passes with zero differences, but complete +source-information acceptance remains RED: five illumination sources across +two sites have 90 NULL cells instead of native values. The original integer +gate detects 40 Frame/Series/Height/Width cells; independent inventory adds +ten Scaling and forty FileName/PathName/URL/MD5Digest cells. Issue #444 is +formally linked to PR #394. No field exclusion or tolerance change is admitted. + +A separate coarse diagnostic on the same source partitions IFC's 13.838053 s +execution into 9.373829 s inside the raw-callable boundary and 4.464224 s +outside. Advanced's 8.087572 s partitions into 3.836250 s inside and 4.251322 s +outside. The callable boundary includes decorated processing and result +assembly, not just numerical kernels. Nested inclusive spans are not added. +Exclusive recording, table assembly and export work has a combined envelope +of 2.812474 s for IFC and 2.882479 s for Advanced. Saving two seconds requires +roughly 71.11% and 69.38% reduction respectively; recording alone is insufficient. +Stack loading costs 0.126811 s / 0.106307 s. Ordinary progress events bound +between-step/axis work below 17 ms and other job-minus-axis gaps at +0.196432 s / 0.100492 s. Those routes cannot close the multi-second target. +Diagnostic clocks are not ordinary speedup evidence. + +Retained evidence lives under ``/var/tmp``: ordinary outputs and source freeze +in ``openhcs-latest-ordinary-advanced-ifc-20261002``, fresh native inputs, +reports and IFC comparison in +``openhcs-current-native-advanced-ifc-v2-919fee-20261002``, and coarse regions +in ``openhcs-current919-runtime-phase-probe-20261002``. Independent source-field +inventory is ``advanced-fresh-native-missing-illum-metadata-20261002.json``; +the dominant cost/owner receipt is +``openhcs-919-measurement-lifetime-frontier-20261002.json``. Original failed +native preparation and the exact metadata comparison failure remain retained. +Actual typed measurement-request/export capture is in progress; no production +optimization or replay acceptance follows from the cost envelope alone. + +Latest main ``3df650bf7`` (merged #442/#443) is integrated at ``436e49755``. +The incoming BioFormats calibration, MCP streaming and existing publication/ +opaque-domain controls pass: 141 tests in 16.72 seconds. The initial narrow +launch failed collection because the MCP test imports the streaming fixture +module by its short name; the corrected command includes that actual fixture +owner first. Both logs are retained in ``/var/tmp``. No source/test guard was +modified to obtain the pass. + +The actual IFC capture completed the public pipeline and exact source/dependency +freeze, but its four-snapshot admission remains RED: step22 never invokes +MeasurementsOutputRecorder.record. Only the terminal spreadsheet before/after +graphs were captured (44,935,114 and 56,109,057 bytes), with no graph or budget +rejection. The nine real recorder calls publish identification/filtering/ +expansion/relationship core tables. Texture and intensity-distribution tables +use the separate per-object recorder. Thus the proposed capture and the premise +that all outer recording cost is redundant wide-feature conversion are rejected. + +Independent export-only qualification passes both unchanged V6 loads (615 arrays +each), exact original CSV bytes, existing strict CP science, and all determining +field/source/array aliases. Thirty-five immutable metadata owners drop their +derived views by their existing declared transport policy; every populated view +matches fresh recomputation from the same fields. Original raw-heap equality +remains RED for that intentionally untransported cache state, separately from +declared-state equality. The actual exporter graph has 31 measurement tables +and two relationship records, with no label pixels; it cannot validate a dense +label-storage optimization. Local diagnostic profiling finds repeated row +identity/axis scanning and assignment; it is not end-to-end speedup evidence. +The ordinary export ceiling is 1.4077 seconds, so export alone cannot save two +seconds. Preserving the native CSV span leaves an optimistic combined outer +recording/export ceiling of 2.4189 seconds; a two-second saving requires about +82.68% collapse. No implementation is admitted by that ceiling alone. +Receipts are ``openhcs-measurement-export-declared-state-profile-20261002`` and +``openhcs-measurement-export-declared-fields-alias-control-20261002.json`` under +``/var/tmp``. The original four-snapshot and raw-heap failures remain retained. + + +Current-main integration and recording attribution (2026-10-02) +--------------------------------------------------------------- + +Main was normally merged through 3cb701770 at 6097ec4ff. Independent PR447 +merged the source-only CPA metadata repair and closed issue444. Its own +220 controls, original architecture check, public Advanced run and independent +scientific/source-identity comparison are documented in +``cpa-secondary-source-metadata-444.rst``. The larger branch still has open +acceptance gates and remains draft. + +The main58ee singleton controls exposed a real prepared-ABI integration bug: +unannotated COMPOSED functions lost their nominal carrier. d4e800a75 derives +that default from the existing consumption declaration while preserving +explicit ndarray/Any/nominal ABI precedence. 136 isolated and249 integrated +controls pass; the original scoped architecture check has zero increases. + +A fresh frozen746 Advanced run completed with compilation1.7983s, +execution9.4002s and total12.3852s, default observers and startup excluded. +Its scientific database comparison has zero differences. All90 previously +NULL calibration fields are present;70 match literally and20 native staging +PathName/URL strings are proven to identify the same source files using +same-inode, resolved-source and original native input-inventory SHA checks. +The raw path-string comparison remains separatelyRED. This is one observation, +not a performance benefit: colocalization and one primary identification step +were slower than frozen919, while database export changed1.8378 to1.7538s. + +The first nine-recorder cProfile diagnostic is scopeRED. It omitted the +existing monitoring callback thread filter and attributed background sleeps +to arbitrary numeric callers. A separate companion-thread experiment +reproduced the error and confirmed the existing worker-profiling policy +removes it after every profiler enable. No production workaround or duplicate +issue for the already-fixed profiler infrastructure was introduced. + +The corrected frozen919 diagnostic has zero foreign sleep/server/poller +events, exact original IFC CSV bytes, and zero scientific differences against +native CP for1800rows. Its disjoint recording attribution is diagnostic only: + +================================ ============ +Region Profile time +================================ ============ +Centroid calculation 0.6766s +Sparse relationship row assembly 0.4404s +Repeated axis-domain scans 0.2159s +Other table assembly 0.3644s +Ownership validation 0.000027s +Measurement storage 0.0123s +================================ ============ + +Centroid-to-IJV conversion is nested within centroid calculation and must not +be added to it. Storage/ownership are immaterial targets for these recording +calls; centroid-only cannot deliver the multi-second goal. Recording plus +nonserialization export remains an optimistic2.4189s ceiling, not a measured +counterfactual improvement. + +A real source-born3D diagnostic on isolated8dec failed mask alignment at +step18. It observed300 source births and18 completed producer groups with +no observation errors. RescaledDNA starts with60 correlated physical Z rows; +Resize preserves60planes and changes onlyXY but drops the source domain. +Resize after ImageMath repeats the loss. Source audit identifies bare ndarray +annotations on the public Resize entries despite their shared implementation +requiring metadata/masks. Direct unwrapped tests bypassed this boundary. +Issue450 is formally linked to PR394; truthful RuntimeArrayData declarations, +prepared-contract tests and a fresh production rerun are required. No source +facts are injected into legacy captures, and no3D speedup/parity claim is made. + +Retained thread-scope/science receipts: +``/var/tmp/openhcs-nine-recorders-profile-thread-scope-control-20261002.json``, +``/var/tmp/openhcs-nine-recorders-profile-thread-owned-science-20261002.json`` +and ``/var/tmp/openhcs-nine-recorders-thread-owned-disjoint-analysis-20261002.json``. +Actual failed3D proof: +``/var/tmp/openhcs-spatial-plane-sourceborn-diagnostic-v2-20261002/observations.json``. + + +Actual 3D production completion and remaining qualification (2026-10-02) +---------------------------------------------------------------------- + +The isolated physical-plane prototype at aba4bfaf3 normally incorporates this +branch through424bd2029. Its truthful Resize ABI declarations and prepared +FULL_STACK controls pass438 tests. The actual new production run completes +with no observer errors,780 source-plane births and32 producer groups. Resize +now retains the runtime stack axis and all60 correlated physical rows; +MonolayerMask and the primary input consequently use the existing declared +stack-binding path. The diagnostic controller had required the different +opaque-image fallback's hook. Its original observation gate remains RED; +zero calls to that fallback is not a production failure or evidence of lost +metadata. The actual route is separately adjudicated from retained producer +observations and the existing binding declaration. + +Independent unchanged saved-output comparison passes all6 CSVs and120 TIFFs +exactly against the specified retained ordinary reference: headers/cells and +image dtype/shape/pixels. Receipt: +``/var/tmp/openhcs-spatial-plane-aba4-independent-historical-science-20261002.json``. +Diagnostic clocks are compilation2.1216s, execution9.7020s and total12.7136s; +they exclude server startup but do not establish a new performance benefit +or a current native CP ratio. Original R1 passes within its unchanged budget; +original R0 retains the quoted-Iterable finding. Independent actual RGB-volume +controls show identical pixels/masks but incorrect XY metadata in the +prototype: (3,3) instead of the declared channel-aware (2,3). Baseline passes. +The separate 2D RGB mask failure also occurs on baseline and is not claimed +as a new regression. + +The decisive route reassessment rejects promotion of the broader physical-Z +prototype. Preserving Resize's runtime stack carrier makes the existing typed +binding sufficient here; the new opaque physical-plane binding has no actual +production consumer in this run. Synthetic controls and successful saved +science do not establish its runtime payoff. Further prototype implementation +is stopped; the next counterfactual applies only truthful carrier declarations +and existing prepared metadata/mask/stack controls to current PR394. Actual +production execution must identify any remaining necessary repair. The +original failed observer gate and the RGB counterexample remain retained; +no source facts are injected and no gate is retroactively called PASS. + +PR394 normally incorporates latest main4754fbe2b at2aa930eb2. The added Napari +integration controls pass183 cases and fail the existing offscreen window +width assertion (1260 versus0). The exact isolated assertion also fails on +clean main4754fbe2b with the same values; it remains a separately recorded +inherited gate failure, not a passing full suite. Logs: +``/var/tmp/openhcs-pr394-latest-main4754-napari-controls-20261002.log`` and +``/var/tmp/openhcs-main4754-napari-layout-isolated-20261002.log``. + + +Qualified narrow carrier repair and fresh native 3D (2026-10-02) +--------------------------------------------------------------- + +The rejected physical-plane prototype is absent from PR394. Narrow commit +b7640ad61 on isolated main-synchronized2aa930eb2 changes only five geometry +annotations; no numerical body, domain, HoleRemoval or renderer changes. +Nine prepared/FULL_STACK controls establish7 failures before and9 passes +after, preserving typed masks, runtime/opaque domains and source provenance. +The related suite passes148 tests; unchanged original R0/R1 both pass within +their original budgets. Integration fecf3114a has exactly the same production +and test Git trees as the qualified isolated source; only validation prose +differs. The fix is formally carried by PR394 for issue450. + +The actual uninstrumented default3D pipeline completes, with exact6CSV and +120TIFF saved science against the same retained reference. All180 actual +source-plane references belong to exactly3 physical TIFFs with indices0..59 +and unchanged source-file SHA values. Both fresh native CP4.2.8.1 measured +repeats pass the existing strict6table/150key/2825fact comparison, exact two +logical uint16(60,256,256) label volumes and complete physical-file coverage. +Native comparison to the prototype plus exact prototype/reference/narrow +comparisons establishes transitive narrow parity; no direct narrow/native +comparison is claimed. The original comparison-controller API failure +(tuple subtraction from frozenset) remains RED; a separate recipe supplies +the required frozenset input views without changing any scientific gate. + +Fresh current main4754fbe2b also matches the same saved reference exactly. +All clocks below use CPU5 and one thread, with startup/warmup excluded. +OpenHCS includes default OUTCOMES and memory observation; native invocation +includes prepare_run, module execution, post_run and Measurements.close, +and excludes imports/JVM/pipeline loading plus its separate warmup. + +===================== ============ ============ ============ +Observation Compilation Execution Total +===================== ============ ============ ============ +Main4754 ordinary 1.6754s 10.0712s 12.4766s +Narrowb764 ordinary 1.6190s 8.3008s 10.7417s +Native measured0 excluded 14.3544s invocation +Native measured1 excluded 14.4375s invocation +===================== ============ ============ ============ + +The observed main/candidate difference is1.7704s execution and1.7349s total +for the full PR394 plus annotation repair. It is not evidence that the five +annotations alone save that time. One ordinary observation per source does +not establish repeatability or statistical significance. Native's two-run +mean is14.39595s, giving a descriptive1.7343x candidate execution ratio and +1.3402x whole-pipeline ratio under the explicitly different clock scopes. +The target gap remains: reaching2x requires about1.10s further execution +reduction, or3.54s whole-pipeline reduction. No goal completion is claimed. + +Receipts: +``/var/tmp/openhcs-geometry-narrow-original-guards-20261002.json``, +``/var/tmp/openhcs-geometry-narrow-b764-3d-ordinary-20261002/observations.json``, +``/var/tmp/openhcs-geometry-narrow-b764-independent-science-and-inputs-20261002.json``, +``/var/tmp/openhcs-main4754-independent-historical-3d-science-20261002.json`` and +``/var/tmp/openhcs-native-3d-aba4-qualification-20261002/fresh-native-scientific-comparison-supplemental.json``. + + +Four-run 3D comparison and dominant runtime frontier (2026-10-02) +---------------------------------------------------------------- + +The subsequent A1/B1/B2/A2 ordinary comparison supersedes the single-pair +ratios above. A is frozen clean main4754fbe2b; B is frozen narrowb7640ad61, +including the full PR394 runtime change and five truthful annotations. Each +run uses CPU5, one worker/thread, the unchanged public driver, default OUTCOMES +and memory observation. Mandatory server/library/kernel startup and shutdown +remain outside the pipeline clocks. All four observations preserve exact six +CSV tables and120 TIFFs against the same retained reference, unchanged source +and dependency freezes, and all180 source-plane references to the same three +physical volumes used by fresh native CP. + +======================= ============ ============ ============ +Observation Compilation Execution Total +======================= ============ ============ ============ +Main A1 1.6754s 10.0712s 12.4766s +Candidate B1 1.6190s 8.3008s 10.7417s +Candidate B2 1.7936s 8.7713s 11.3281s +Main A2 1.6213s 10.0910s 12.4374s +Main mean 1.6484s 10.0811s 12.4570s +Candidate mean 1.7063s 8.5360s 11.0349s +======================= ============ ============ ============ + +The descriptive mean reductions are1.5451s execution and1.4221s total. +Native warm invocation mean14.39595s gives1.6865x execution and1.3046x +total ratios under the previously stated clock scopes. Two observations per +source do not establish statistical significance or annotation-only causality. +The remaining2x gap is1.3381s execution and3.8369s total, larger than the +first-pair estimate. The complete independent receipt is +``/var/tmp/openhcs-3d-b764-abba-science-clocks-20261002.json``. + +A separate source-pinned diagnostic partitions8.3448s of execution into +4.0497s inside the callable boundary and4.2951s outside it. Callable time +includes decorated metadata/control work, not only numerical kernels. The +exclusive outer terms include stack loading0.5991s, unstacking0.2126s, +saving0.3487s, output identity0.2070s across1680 calls, image recording0.2333s, +CP image requests0.2012s, SaveImages invocation preparation0.3463s, metadata +finalization0.1112s, publication0.4795s and reconciliation0.2812s. These are +diagnostic spans, not accepted speedup measurements; required copying, +validation and I/O remain within them. + +The SaveImages preparation span stores memory artifacts and contextualizes +their values; it is not TIFF encoding. Actual artifact materialization has a +separate0.0831s envelope. Saved metadata contains120 SourceArtifactProjection +entries from the two SaveImages outputs. The generic main-flow metadata/VFS +reload branch is not the observed publication route. The actual joint route +is artifact invocation/store, successful materialization metadata, source +artifact projection, serialization, locked publication and reconciliation. + +Publication/finalization/identity alone has an optimistic1.0789s ceiling and +is rejected as insufficient to close the current execution gap. The broader +cohort/recording/invocation/publication envelope is3.0201s and requires at +least44.3% collapse before accounting for mandatory work. Existing whole-stack +cache lookup/storage is already below1ms; dictionary tuning or assuming every +input is restacked cannot support the target. The next admission requires an +actual saved joint leaf replay preserving metadata mutation, source correlation, +custom normalization, error ordering and pixel/mask isolation. No new mutable +identity cache or production optimization has yet been admitted by this ceiling. + +Diagnostic science and freeze validation: +``/var/tmp/openhcs-narrow450-phase-science-and-freeze-20261002.json``. +Global authority/consumer inventory and corrected route: +``/var/tmp/openhcs-b764-publication-authority-consumer-inventory-20261002.json`` +and ``/var/tmp/openhcs-b764-coupled-publication-source-frontier-v2-20261002.json``. + +Latest mainb68029c0c is normally merged atc0d78b62e. The initial MCP/agent +integration run passes131 cases and fails both MRO variants of a new cold +feedback fixture. Clean mainb680 reproduces both failures: its reporting thread +can observe published progress before the fake compiler sets its subsequent +``emitted`` marker. This identifies a fixture synchronization race, not a +demonstrated production race. The original candidate and clean-main failures +remain retained. Baseline causal receipt: +``/var/tmp/openhcs-main-b680-cold-feedback-race-baseline-20261002.json``. +Benchmark timings above remain pinned to their actual main4754/narrowb764 +sources; the newer MCP merge is not silently included in their evidence. + +Test-only integration3317bf9c2 sets the compiler-entry marker before publishing +progress. The actual reporting callback still releases held work; the original +three-second timeout, both cooperative MROs, ContextVar restoration, terminal +ordering and original compiler-error assertions are unchanged. No production +race workaround, retry or longer timeout is introduced. The clean-main repair +passes124 cold-feedback/agent-service controls; integrated PR394 then passes +all133 controls including the narrow prepared geometry ABI tests in13.72s. +Both initial failed runs remain retained. Integrated command, CPU3: + +.. code-block:: bash + + env PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/home/ts/code/projects/openhcs-shared-runtime-plumbing \ + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \ + NUMBA_NUM_THREADS=1 CUDA_VISIBLE_DEVICES='' taskset -c 3 \ + /home/ts/code/projects/openhcs/.venv/bin/python -m pytest -q \ + -p no:cacheprovider tests/unit/agent/test_cold_inspection_feedback.py \ + tests/unit/agent/test_agent_services.py \ + tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py + +Pass log: +``/var/tmp/openhcs-pr394-main-b680-repaired-integration-controls-20261002.log``. + +Saved-leaf rejection and whole-worker diagnostic, 2026-10-02 +---------------------------------------------------------- + +A second source-b764 diagnostic captures the actual contextualization, +normalization scopes, identity-cache preimages, serializer inputs and five +locked publication preimages. All39 admitted snapshots retain their original +metadata, source correlations and alias relations. Exact six CSV/120 TIFF +comparison against the same-head ordinary run and retained historical reference, +all180 physical input references and source/dependency/native freezes pass. +Injected capture work is separately timed; these are not accepted pipeline clocks. +Receipt: +``/var/tmp/openhcs-narrow450-joint-leaf-v3-science-and-freeze-20261002.json``. + +An exploratory metadata projection fusion passes the11 captured valid-input +leaf comparisons but saves only4.82ms across the five actual context calls; +normalization has no measured gain. Separately, malformed self spatial-domain +metadata combined with malformed source fallback spacing changes the first +exception. The original reports the spatial-domain failure; the candidate +reports invalid spacing. This is a real correctness failure, independent of a +historical test that observes normalization phase counts. The candidate is +rejected for both inadequate payoff and error-order drift, and is not included +in this branch. Original/candidate replay receipts: +``/var/tmp/openhcs-narrow450-joint-leaf-v3-original-replay-20261002.json`` and +``/var/tmp/openhcs-narrow450-joint-leaf-v3-fusion-replay-20261002.json``. +Malformed-input witness: +``/var/tmp/repro_narrow450_fusion_domain_spacing_order_20261002.py``. + +All five original locked publication transactions and both serializer calls +replay with exact ordered JSON results and argument after-state. Original replay +transaction medians sum to0.4646s; the observed callback sum is0.3760s and +lock/read/JSON-write residual0.0872s. Replacing three final transactions with one +would remove only about0.04s of this replay's I/O baseline. Standalone batching +is rejected as insufficient. These are saved-input replay clocks, not an +end-to-end speedup; original on-disk whitespace was not captured. Receipt: +``/var/tmp/openhcs-original-publication-v3-replay-20261002/receipt.json``. + +One further actual public-driver diagnostic uses the existing thread-owned +worker profiling policy, with no source edits or diagnostic monkeypatches. +The worker records26,443,742 calls, including5,280 slice-context projections, +25,304 source-metadata merges,55,173 source identity constructions and248 NumPy +stack calls. The next investigation follows repeated whole-image/plane +derivations across loading, invocation, saving and publication. It does not +assume every cache hit copies or every saved plane has independent pixels: +the observed SaveImages named values, VFS planes and cache share pixels, while +the observed Resize output has independent buffers. Profiling increases the +execution clock substantially; profile times must not be scaled into ordinary +costs or reported as benchmark speedups. Exact six CSV/120 TIFF and frozen source, +dependency, native and physical-input comparisons pass. Receipts: +``/var/tmp/openhcs-narrow450-existing-worker-profile-20261002/observations.json`` +and +``/var/tmp/openhcs-narrow450-existing-worker-profile-science-and-freeze-20261002.json``. + +Main17c630606 is normally merged ata6abc4418. Its new lifetime controls initially +produce140 passes, two SDK failures and one teardown error in the integrated +repository suite. The generated child imports PolyStore before activating the +selected OpenHCS checkout; the intentionally failing synthetic callback also +outlives its test into autouse cleanup. Test-only789e3c304 activates OpenHCS first +and restores the callback owner in a lexical monkeypatch context. Original +failure identity, cooperative close, main-thread checks and repeated-close +assertions remain unchanged. All142 lifetime/cold-feedback/agent-service/prepared +geometry controls then pass in25.24s. This is tracked separately by issue457; +closed issue455's installed production shutdown acceptance is not reopened. +Original RED and repaired logs: +``/var/tmp/openhcs-pr394-main-17c630606-integration-controls-20261002.log`` and +``/var/tmp/openhcs-pr394-main17-lifetime-fixture-repair-controls-20261002.log``. +AST/owner receipt: +``/var/tmp/openhcs-pr394-main17-lifetime-fixture-repair-20261002.json``. + +The test-only repair ships independently in merged PR460, commitb495dbb13; +issue457 is formally linked and closed. Its exact clean-main branch passes21 +lifetime/stdio/dev-client controls through normal repository conftest collection +in13.21s. Main is normally merged back into this branch at01875a772; the merge +introduces no additional production diff. Clean-main qualification receipt: +``/var/tmp/openhcs-mcp-lifetime-fixture-457-main-qualification-20261002.json``. + +The next ownership census uses NRA's ModuleSyntaxIndex and canonical +CompactClassFamilyIndex on all703 OpenHCS production modules and the actual +63 PolyStore,12 python-introspect and6 metaclass-registry modules. All5506 +original class declarations remain represented;5493 join canonical family +declarations and13 remain unprojected OPEN. There are no parse errors. +Conditional/function-local binding, omitted numerical dependencies and native +runtime effects remain explicit proof limits. This is source discovery, not +an admitted architecture or equivalence proof. NRA revision0844525ec is pinned. +Receipt: +``/var/tmp/openhcs-b764-shared-derivation-nra-class-census-20261002.json``. + +Current-main integration and acceptance findings, 2026-10-02 +----------------------------------------------------------- + +Main cc9fcdfd4 is normally merged at 133da74ec. The affected native +presentation/MCP, agent-service, resource-lifetime and prepared-geometry suite +passes 150 cases in 17.75s. A separate current-source owner gate passes 73 +metadata-owner, live-mutation, provenance-constructor and resolution-snapshot +cases in 1.34s. These scoped gates do not erase the retained broader offscreen +window failure or qualify a new benchmark source. Logs: +``/var/tmp/openhcs-pr394-main-cc9fcdfd4-integration-controls-20261002.log`` and +``/var/tmp/openhcs-pr394-current-maincc9-source-owner-controls-20261002.log``. + +The original, unchanged whole-branch R0 is rerun against cc9fcdfd4 and +133da74ec with tool revision 3b03785f4. It exits 1. Its only positive measures +remain source_metadata's one TypeSwitch and three TypeSwitchArms; the removed +virtual_workspace_metadata classifier contributes minus one switch and minus +four arms. The ownership decision retains scalar admission on +SourceMetadataFields and durable rejection on DurableSourceMetadata: the +grammar belongs to the field owner, while canonical runtime spelling and +literal durable spelling remain substitutable policies. Reintroducing the +decoder's duplicate validator or moving the grammar to an unrelated utility +would contradict that ownership. This adjudicates the relocation's architecture; +it does not declare the per-file ratchet passed or alter its implementation, +budget, exclusions or observations. Primitive/subclass admission, reported-class +read ordering, scalar/container precedence, exact errors and mutable lifetime +remain separate behavior obligations covered by the owner controls. Raw report: +``/var/tmp/openhcs-pr394-current-maincc9-original-r0-20261002.log``. + +The metadata-only whole-query prototype is rejected immediately after saved +replay: its five captured context calls still take about 55ms, with no matched +speedup demonstrated. Valid saved leaf gates pass, but constructor/subclass, +malformed-input and live-mutation obligations remain unresolved. Neither its +callback-based query abstraction nor its manually replayed constructor effects +are promoted. A separate source trace falsifies wholesale stack-rebuild removal: +23 of 32 loads already hit the whole-stack cache and memory saves retain typed +payload references. The entire load/unstack/save/identity envelope is only +1.3674s before mandatory work, insufficient to close the 1.3381s execution gap. +The next route must span repeated source derivation across existing request, +recording, stack and publication owners. Receipts: +``/var/tmp/openhcs-whole-source-query-exploratory-REJECTED-20261002.json`` and +``/var/tmp/openhcs-b764-whole-cohort-cycle-dominant-route-review-20261002.json``. + +Issue 435's fresh public MCP synthetic acceptance is RED on frozen 133da74ec. +The original five processing steps are unchanged; only owned output paths and +the unused required DAPI binding differ. The first attempt stops before runtime +creation because native TCP lock paths do not follow XDG storage. A fresh, +separately retained attempt admits only its declared private data/control lock +paths, completes mandatory preparation and compilation, then fails the fifth +step at the unchanged duplicate source-projection guard. Its exact owned runtime +is closed through the public API and all SDK children are terminal. This is a +real workflow failure that the existing same-occurrence unit fixture did not +cover; installed acceptance remains open. No original biological replay or +performance claim follows from this synthetic check. Source/input/dependency, +native, request/reply and failure evidence: +``/var/tmp/openhcs-derived-role-435-synthetic-acceptance-v2-20261002``. + +Current main integration and publication diagnosis, 2026-10-02 +----------------------------------------------------------- + +Main 283119c42 is normally merged after the declared independent-plane NLM +implementation and its documentation land. The combined affected NLM, +cold-feedback, agent-service, prepared geometry, native presentation and MCP +resource-lifetime controls pass 168 cases in 25.84s. Log: +``/var/tmp/openhcs-pr394-main-283119c42-integration-controls-20261002.log``. +The qualified benchmark sources and clocks above remain unchanged. + +The separately retained issue 435 public diagnostic reproduces the original +failure. Four actual projections show two correctly qualified main-flow images +but both retained artifacts address the same unqualified TIFF. The final raw +artifact overwrites the capped artifact. The same-occurrence ownership comparator +is never called: destination matching fails before that comparison. The existing +duplicate guard therefore detects a physical filename collision, not a guard +that should be relaxed. Both observer hooks call their original implementation +once; no capture errors are reported. The exact owned runtime is closed through +the public API and all SDK children are terminal. Receipt: +``/var/tmp/openhcs-derived-role-435-collision-facts-v3-20261002.json``. +An isolated repair on existing materialization-purpose and filename owners is +under review; its local test results do not yet establish public acceptance. + +The owned source-field composition prototype is also rejected: the eleven +captured valid leaves and additional mutation/constructor controls agree, but +the five context, three normalization and three identity calls save only 9.24ms +warm or 5.96ms cold. No candidate pipeline qualification or PR is opened for it. +Receipt: +``/var/tmp/openhcs-owned-source-composition-exploratory-REJECTED-20261002.json``. + +SaveImages reconstructs two source stacks solely for materialization metadata, +whose downstream consumer reads provenance. Its earlier 0.346s envelope includes +four contextualization children already measured elsewhere. After separating +those children in the retained diagnostics, the residual envelope is only about +0.10--0.114s. This is rejected as a standalone performance route; it cannot be +counted as an independent 0.346s saving. The complete source-query preimage was +not captured, so resulting metadata leaves are not a source-lookup replay. +Read-only audit: +``/var/tmp/openhcs-b764-save-images-artifact-subtree-readonly-audit-20261002.json``. + +Automatic image-role repair and public execution, 2026-10-02 +---------------------------------------------------------- + +Commits 0b6c66560 and 2d756646b repair the physical issue 435 collision on +existing owners. Artifact type and materialization purpose supply the retained +role; shared filename identity supplies parser-backed names. MaterializationBatch +binds the original rendering spec onto its existing context, because a public +caller may render a spec different from the compiled plan's default. The original +output plan and source references remain intact. Explicit exports keep their +authored paths. Both scalar and projected retained images preserve physical +coordinates and complete filename suffixes. Missing parsers use the existing +required-parser error policy. Strict projection and path-conflict guards remain +unchanged. + +The initial candidate's two actual-purpose mismatch controls fail before this +binding repair. The initial scoped R0 also reports a foreign optional-plan probe; +the corrected owner eliminates that probe without changing the guard. Original +scoped R0 and R1 pass for the complete repair versus 51533665e. Both original +failures are retained. Related controls pass 171 cases; after normal main +283119c42 integration at 053c9d445, 183 affected controls pass. An attempted +unrelated Torch NLM control stops at collection because Torch is absent; that +failure remains separate and no dependency changes follow. Handoff: +``/var/tmp/openhcs-automatic-image-role-435-reviewed-handoff-20261002.json``. + +The clean frozen 053c9d445 source completes public MCP startup preparation, +compilation and all five original processing steps. The output contains exactly +the two distinct qualified role TIFFs and two matching source-projection rows. +The fourth journey then stops at its inspection assertion: the API reports +PARTIAL with the sole missing-grid warning, because the synthetic declaration +does not supply plate grid dimensions. This original whole-journey RED is +retained; it is not a pixel-comparison failure or a completed fresh-process +readback acceptance. The runtime and all SDK children are terminal. Receipt: +``/var/tmp/openhcs-derived-role-435-synthetic-acceptance-v4-20261002/journey.json``. +Supplemental public inventory, complete pixel and fresh-process checks remain +pending on these unchanged outputs. + +The reviewed source is normally integrated into PR394 at fa694eb56. Its affected +current NLM, cold-feedback, agent-service, prepared geometry, native presentation, +MCP lifetime and new role controls pass 194 cases in 20.43s. Log: +``/var/tmp/openhcs-pr394-automatic-role-435-current-integration-controls-20261002.log``. +Neither this bug repair nor its public synthetic run establishes a performance +gain. Accepted benchmark clocks remain pinned to main4754 and candidateb764. + +The global runtime reassessment finds no sufficient measured optimization route +yet. Removing 485 provenance births and 485 identity births in captured context +leaves already saved only 4.82ms and changed malformed-input ordering. The next +candidate must remove whole repeated projection/adapter transactions, rather +than another constructor or field-query leaf. The strongest remaining joint +frontier is 3.0201s with incomplete whole-lane replay coverage; counts from the +instrumented profile cannot establish its reducible fraction. Audit and smallest +missing capture recipe: +``/var/tmp/openhcs-b764-whole-lifecycle-architecture-reassessment-20261002.json``. + +Issue 435 supplemental public readback passes on the unchanged V4 output. Two +fresh SDK processes expose identical complete inventory, sample and projection +facts. Both 1024-by-1024 float32 role images pass the unchanged existing CP pixel +comparison with zero out-of-tolerance pixels: raw maximum difference is zero; +capped maximum difference is 1.1920928955078125e-7 under atol=rtol=1e-6. Both +roles retain physical C2 and calibration within the existing inspection spacing +tolerance of 1e-12. The original controller, journey, inputs, source, dependencies, +native binaries and all physical output hashes remain unchanged, and both reader +children exit zero. The original V4 journey remains RED. The supplemental +inspection check follows the existing status policy, retaining PARTIAL for the +sole undeclared-grid warning without fabricating grid metadata; spacing uses the +existing inspection tolerance rather than literal equality after TIFF formatting. +This is synthetic source qualification, not original biological or installed-root +acceptance. Supplemental receipt: +``/var/tmp/openhcs-derived-role-435-v4-readback-supplement-20261002/receipt.json`` +(SHA256 4f0f0550e50aaff297d134ee5accf2b2da01297305394a155fe79e827b844fdf). + +Current source qualification and workload priority, 2026-10-02 +------------------------------------------------------------ + +Main 142196857 (callable documentation ownership) is normally merged at +9aa3918d2. The affected declared-documentation, agent-service, cold-feedback, +automatic-image-role and plane-NLM integration controls pass 170 cases in +14.77s. Log: +``/var/tmp/openhcs-pr394-main142-documentation-role-integration-controls-20261002.log``. + +A clean frozen 9aa3918d2 ordinary public-driver observation completes one CPU5 +inline worker/thread with compilation 2.103262s, execution 7.870635s and total +10.781240s. Server/library/kernel startup and shutdown remain outside pipeline +clocks. All six CSVs, 120 TIFFs, 180 disk projection references and three source +volume hashes agree exactly with the retained qualified reference and b764 +output; source, dependencies, interpreter and native binaries remain unchanged. +Native science is transitive through that exact reference. This single current +observation does not replace the earlier repeated ABBA comparison or establish +a new optimization gain. Receipt: +``/var/tmp/openhcs-pr394-role-qualified-3d-science-20261002.json``. + +The descriptive comparison against the two retained warm native 3D observations +(mean 14.395949s) is 1.829x execution and 1.335x total. 3D is weakest among the +three qualified representatives (3D, Advanced and IFC), whose OpenHCS revisions +differ. The global weakest case remains unknown: all30 cached native clocks +include subprocess startup and the older multicore CP scaling baseline is +projected. Historical WoundHealing, TrackObjects and worm cases require current +matched warm-native qualification before ranking the whole catalog. Receipt: +``/var/tmp/openhcs-current-native-workload-priority-revalidation-20261002.json``. + +The next whole-transaction diagnostic is rejected before launch when its saved +representative identity pairs predict 2,719,141,800 bytes for three 60-request +cohorts alone, exceeding the transport's 1GiB root budget. No pipeline is run +for V4 and none of its frozen files are changed. This prediction is not an +observed complete production roster. A saved grouped-graph experiment reduces +those three representative graphs to 81,450,818 bytes while preserving actual +saved payloads; cache subsets contain no arrays. A separate V5 preparation +must capture actual ordered requests and detached pre/post cache states before +its replay or a performance candidate can be admitted. Receipts: +``/var/tmp/openhcs-whole-cohort-v4-budget-preflight-REJECTED-20261002.json`` and +``/var/tmp/openhcs-grouped-identity-budget-saved-experiment-v3-20261002/receipt.json``. + +Fresh native qualification changes the primary target +--------------------------------------------------- + +Four additional ordinary current9aa cases now have matched source inventories +and actual warm native runs. Each native case excludes one warmup and retains +both measured repetitions. OpenHCS uses one CPU5 inline worker/thread, mandatory +READY warmup before compilation, the default OUTCOMES observer and memory policy. +Server startup and shutdown are excluded. The following ratios are descriptive: +one OpenHCS observation against the mean of two native observations, not a +repeated A/B optimization claim. Native time includes preparation, modules, +post-run and measurement closure, excluding imports, JVM startup and pipeline +loading; OpenHCS total includes compilation. + +.. list-table:: Current single-worker observations, seconds + :header-rows: 1 + + * - Case + - OpenHCS execution + - OpenHCS total + - Native mean + - Execution speedup + - Total speedup + * - WoundHealing + - 3.571737 + - 4.096850 + - 3.779188 + - 1.058x + - 0.922x + * - TrackObjects + - 5.036489 + - 6.400329 + - 8.276148 + - 1.643x + - 1.293x + * - UntangleWorms + - 2.194352 + - 3.206172 + - 2.974680 + - 1.356x + - 0.928x + * - UntangleWormsBrightField + - 3.064656 + - 4.264850 + - 6.152021 + - 2.007x + - 1.443x + +All eight comparisons pass the existing measurement/schema/discrete gate and +unchanged numeric tolerances. WoundHealing and BrightField also pass complete +saved-output science. TrackObjects and UntangleWorms fail saved-image content +comparison in both repetitions; their ratios therefore have measurement-only +qualification. Physical output inventories match. All original RED receipts +remain unchanged. TrackObjects saves 21 three-panel PNGs: original-image and +outline panels agree exactly, while the tracked-object panels are gray instead +of the native colored labels and white IDs. The current callable deletes the +requested saved-image options and returns its unchanged input image. Worm +images have matching backgrounds and colors but 96 differing outline pixels +across two outputs. Sparse-overlap rendering is a candidate cause requiring +actual producer inputs, not a proved segmentation diagnosis. + +WoundHealing is weakest among seven retained representative measurement-science +scopes, replacing 3D as the primary investigation. This is not a full-catalog +ranking. WoundHealing needs about 1.682142s execution reduction and 2.207256s +total reduction to reach twice its native throughput. Its native comparison +uses exactly two selected 2304-by-1536 RGB JPGs; TrackObjects uses all 21 selected +sequence frames, and both worm cases retain their actual source bindings and +training inputs. Source, dependencies, interpreter, native binaries and all +original inputs are frozen before and after. Evidence: +``/var/tmp/openhcs-current-slowcase-priority-and-png-audit-20261002.json`` +(SHA256 c84d5c0f39bf2a246e00eb7cbb42a7457849efb6a974a6d78fee0e68c6c270ef). + +The existing worker-owned profile on current9aa WoundHealing preserves exact +ordinary CSV outputs. It attributes 0.604904s to 22 runtime stack operations, +0.551025s to 16 image normalizations, 0.288266s to two centroid queries and +0.464524s to two color reductions. These profile durations are diagnostic costs, +not measured savings. Metadata composition itself is only 0.016512s. Source +recording's 0.251105s overlaps normalization and must not be added to it. Even +eliminating stacking, normalization and centroid costs entirely gives an +optimistic 1.444195s, below the execution gap before mandatory copies. Another +small metadata leaf is rejected as a sufficient route. Receipt: +``/var/tmp/openhcs-current9aa-wound-profile-dominant-frontier-20261002.json``. + +The bounded actual-input normalization capture completes successfully: all 16 +calls are recorded, with two unchanged V6 before/after graph pairs for the +large integer arrivals. Both consume the same two-frame uint8 RGB stack, +shape (2, 1536, 2304, 3), scale 255 and float32 target. Original input pixels +and public metadata remain unchanged; source/environment/native freezes pass. +Capture clocks are not performance evidence. Independent whole-output science +and original-leaf replay remain separate gates. Receipt: +``/var/tmp/openhcs-wound-intensity-leaf-capture-v1-20261002/observations.json``. +The generic consumer investigation must preserve exact compiled artifact +selection, source projection and stack-broadcast laws; raw integer pixels cannot +be relabeled as normalized unit-interval pixels to avoid a conversion. + +The former 3D grouped-cohort V5 diagnostic remains prepared and unlaunched after +this priority change. No rejected route is promoted because its preparation is +already complete. The tracking renderer is being investigated in an independent +main142 worktree with actual 21-frame producer inputs and exact native panel +replay; no saved-image repair is included or claimed here yet. + +Library readiness repair and latest qualification, 2026-10-02 +------------------------------------------------------------ + +Main 3f955d3fb (MCP sampling) and cf5a83f89 (fixture CLI repair) are normally +merged. The MCP/prepared-geometry/automatic-role integration gate records 354 +passes and nine failures. All nine fail identically on clean main3f955 with the +same readonly native-binary bindings: eight fixture declarations omit the +required server, and one expected stream-argument mapping omits display_config. +The first clean-main attempt refused a missing native extension at collection; +that original failure is retained before the exact-bound native supplement. +These are existing fixture failures, not a passing full suite. Numerical and +dependency sources are unchanged by the fixture-CLI merge. + +The actual Illumination3 first run reached an unprepared masked polynomial +specialization. Its Numba artifacts were written inside the measured polynomial +step: 0.526s versus 0.016s when already cached. The existing callable preparation +hook now prepares masked/unmasked and readonly canonical image/mask signatures +before READY. A fresh-cache subprocess forbids all later compilation and cache +loads across 36 dtype/layout/mutability/mask combinations and verifies an +independent polynomial oracle, input isolation and geometry errors. Twenty +preparation/fitted-illumination controls pass, with two existing skips. The fix +ships independently in merged PR474, formally closing issue473; original R0 +passes all three roots and original R1 passes its unchanged 160-second budget. +Main a6c18f054 is normally merged back into this branch. Its full receipt is +``docs/validation/polynomial-library-readiness-473-20261002.rst``. + +The clean frozen integrated source54708367b has fresh ordinary CPU5 one-worker, +one-thread observations using the existing cache, default OUTCOMES observer and +default memory policy: + +.. list-table:: Fresh post-repair observations, seconds + :header-rows: 1 + + * - Case + - Compilation + - Execution + - Pipeline total + * - Illumination3 + - 0.962373 + - 0.647611 + - 1.961545 + * - WoundHealing + - 0.346341 + - 3.916717 + - 4.575604 + +Startup/library/kernel preparation and shutdown remain outside pipeline clocks. +These are single observations, with no matched repair speedup or statistical +regression claim. Wound is slower than its retained earlier single observation; +that result is preserved rather than substituting the better sample. Both +authored Illumination3 NPY images and Wound's complete two-row Image.csv agree +byte-for-byte with the qualified unchanged references. Existing complete saved +inventory and CP-tolerance comparison gates pass. Native science is transitive +through those fully qualified references. The first validator incorrectly +expected two Wound tables rather than two rows; its failure remains retained, +and the V2 supplement checks the actual authored inventory without changing any +production comparison policy. Source, dependencies, interpreter, native binaries +and all inputs remain frozen. Receipt: +``/var/tmp/openhcs-pr394-masked-prewarm-ordinary-science-v2-20261002.json`` +(SHA256 21231dffa14dbe17bf824a4258f677e7d656122036da81e0b3d8c9c8bc2f9c02). + +Additional native qualification and remaining execution frontier +--------------------------------------------------------------- + +Current9aa Vitra uses one selected joined source set containing all four physical +resources, rather than two channel-derived image sets. Its two warm native +repetitions average 3.044868s; OpenHCS is 1.615775s execution and 2.908514s total. +Complete saved science passes for 2,233 table rows and its authored image in both +native repetitions. Earlier source-set controller refusals remain retained. +Receipt: +``/var/tmp/openhcs-current-vitra-native-timing-science-v3-20261002.json``. + +Illumination3's two warm native repetitions average 0.603076s. Its authored +pipeline saves two NPY images and no CSV. Complete existing inventory and image +science pass in both repetitions. The retained first OpenHCS observation is +1.309110s execution and 2.603767s total, containing the late polynomial +specialization described above; the fresh post-repair observation is recorded +separately. Cache-state differences prevent treating these as a matched repair +speedup. The global weakest full-catalog case remains unknown. Receipt: +``/var/tmp/openhcs-current-illumination3-native-timing-science-v1-20261002.json``. + +Wound's actual worker profile partitions 3.387111s into 1.536850s of disjoint raw +processing and 1.850261s of other runtime/recording work. The 22 stack operations +include 0.4036s in 16 aggregate image stacks, 0.1481s in object variants, 0.0538s +in main-flow/cache and 0.0023s in initial loading. Ten source-record +normalizations account for 0.2436s of the 0.5510s normalization envelope. These +are profiled costs and overlap the earlier source-query attribution; none are +accepted saved seconds. + +Actual Wound ON_DEMAND QA images are not forced by terminal OUTCOMES publication. +However, raw QA planes borrow object-variant pixels and shared validity masks; +their current aggregates establish independent mutable artifact buffers. +Deferral with equivalent eager per-plane copies cannot remove those copy bytes, +and zero-copy adoption without an ownership/lifetime proof is unsound. Existing +ArrayBridge geometry validation also forces concrete arrays at payload +construction. No lazy wrapper or second mutable identity cache is promoted. +The global AST owner/lifetime audit retains a bounded actual alias-capture plan: +``/var/tmp/openhcs-current9aa-wound-copy-lifetime-reassessment-20261002.json``. + +An external shared-geometry convex prototype preserves both actual saved input +graphs exactly. The outer 256-level call's warm local median changes from +0.260906s to 0.023542s, saving 0.237364s; the nested 16-level call overlaps it +and is never added as independent work. This is useful local evidence, not a +production pipeline gain. The nominal owner audit identifies existing shared +column-envelope and Bresenham authorities and a disconnected 15-kernel legacy +component; production migration and end-to-end qualification remain separate. +The shared dense-coordinate prototype also has exact actual Wound/Track input +replays and passing scoped R0/R1 after removing duplicated admission logic. +Its pre-domain allocation behavior for very sparse large label IDs remains +under review. Neither prototype is integrated here or counted as a speedup. + +Independent tracking PR468 is synced to main and passes 19 focused tests plus +unchanged scoped R0/R1. It remains draft: native saved-image differences shrink +from 77,726 to 3,340 text pixels, but full native PNG parity is still RED. It +formally links issue467. No parity gate, tolerance, scientific output or renderer +environment is weakened, and no tracking repair is included in this branch. + +Remaining acceptance still includes installed-root issue419 IPO/Shape and +Morph/DISTANCE consumers; issue433's original 13.36GB OOM allocation owner and +repeated READY behavior; installed-root issue435; and issue450's installed +prepared-Resize consumer. The original whole-branch scalar-classifier R0 RED +remains separately retained. These draft closing references do not establish +completion. Full native scaling, all30 benchmark reruns and fresh figures remain +outstanding, and the long-running performance goal stays active. + +Shared geometry release and coordinate-owner checkpoint +------------------------------------------------------ + +Main ``5b7f3c48d89da4415de001375d0e06de881153c9`` is normally merged. +Independent PR476 ships the shared quantized hull/Bresenham geometry and +officially closes issue475. Earlier hull issues158/268 and PR161 remain +completed. The full-source PR394 ABBA comparison pins eb2 versus b470: +BrightField total4.635292249 ->3.294145465s, Untangle3.234328632 ->2.311149861s, +Illumination1.988957128 ->1.892348221s. All16 complete saved outputs pass exact +science and source/env/native/input freezes. Wound is a variance control, not +an attributed hull saving. See the committed shared-quantized-hull report for +every sample, readiness/source guards and retained native-image RED. + +The existing object-label storage family now owns the single shared dense +coordinate reducer. Tracking and relationships inherit/use its primitive; +two duplicate kernels are removed. Resource/domain-effect and C/F/A readiness +controls pass. Isolated100 and integrated101 tests pass, both with2 existing +skips; original scoped R0/R1 pass unchanged against eb2. Frozen b470 versus +6e69 ordinary ABBA Wound total3.866369809 ->3.562642806s; Track execution +5.338271976 ->5.052027345s, but total6.629491309 ->6.684317434s regresses +with higher compilation. All8 full saved outputs/freeze gates pass. The separate +shared-dense-coordinate report retains all samples, no causal late-JIT claim +and the original tracking image RED. No reduction resolves issue433's original +allocation owner or the whole-branch scalar-classifier R0 RED. + +Owner WT remains /home/ts/code/projects/openhcs-shared-runtime-plumbing. All +production edits are committed here. Agent-owned implementation branches are +integrated; no additional unpublished production files are claimed. Frozen +benchmark WTs remain unchanged. Compared with receiving checkpoint eb2, new +production edits are precisely three CP geometry/illumination modules plus +core/runtime_object_labels.py and four CP backend/preparation/tracking/ +relationships modules. These are indirect numerical/coordinate and preparation +changes beyond the receiving review's original four-file eb2/wheel delta. +Installed receiving obligations419/433/435/450, whole-branch R0, full native +image parity, full-catalog/scaling and figures remain open. A first-wave fresh +four-case ordinary/native qualification is in progress, not a completed gate. + +Library readiness and saved-reader checkpoint, 2026-10-02 +------------------------------------------------------- + +Main ``b4405ff3541add6b91004002b7d6f389f1208636`` is normally merged. +Independent PR482 closes issue477 and PR486 closes issue481. Main's singleton +invocation fix passes 401 controls and original scoped R0/R1; its actual public +translocation observation is 1.679128s execution / 3.094840s total / 1.008657s +compilation. Both saved warm native repetitions pass full database, schema, +discrete and numerical measurements, properties, exact image and physical +inventory gates. This single main-based observation is not a measured gain. +Receipt: /var/tmp/openhcs-singleton-label-main-477-saved-native-science-20261002.json +(SHA256 5c5c4c497df1ee4b115aba42650941f451fac3ae3f784cfcfe3144e0ae94d6ac). + +Issue481 prepares canonical masked float32/float64 declumping signatures through +the existing morphology backend library lifecycle. Numerical bodies are +unchanged. Eight isolated controls and original scoped R0/R1 pass; cold and +disk-cache witnesses reject subsequent compilation or cache loading. Frozen +integrated ``adef79b4ff3825b35a41ef96b60f3bc421a5ddd5`` Speckles execution is +1.915174s, total 3.212164s, compilation 0.959121s. No kernel-cache writes occur +inside execution. All three scientific CSVs match the original fully native +qualified output exactly, with complete inventory and freezes. This necessary +readiness repair does not establish a speedup. Receipt: +/var/tmp/openhcs-pr394-morphology-ready-speckles-science-20261002.json +(SHA256 29526b6c5c37bbd840c706393b9290695894bc80a2247aedc7c718cb58ee5477). + +Issue479's saved relationship reader is integrated from ``2de4e021f``. It +validates directed edge declarations, object/image domains, reverse pairs and +redundant Parent/Children fields before using scalar projections. Saved +snapshots and cache transport retain endpoint correlations; typed value-only +snapshots explicitly retain UNKNOWN evidence. Both-root scoped R0 passes, +but original R1 remains RED for an unchanged validator's relocation (+1 core, +-1 interop, zero global type-check change). This is not all-guards admission. +The merged-parent controls pass 380; root integration passes 235. + +Corrected saved Beginner replay uses the actual complete output root. Both +native repetitions have exact two-image agreement, complete physical-file +coverage and five matching correlation keys / 2,093 directed pairs. Full +measurement parity remains RED: 80 extra Costes features and 63 missing Nuclei +child-mean features. The earlier helper's incorrect results-directory root and +missing-image report are rejected, retained harness evidence. Reader receipt: +/var/tmp/openhcs-relationship-479-full-saved-replay-v2-20261002/receipt.json. + +Issue487 isolates the first loss for unwanted Costes work: existing settings +postprocessing treats legacy Run all metrics=Accurate as true and overwrites +the five authored flags. Native CellProfiler uses this aggregate setting only +for UI visibility, including when Yes; individual choices remain authoritative. +The correct existing owner is ModuleOnlySettingBinding, with the obsolete +runtime override removed. Public-importer RED and native/global AST census are +retained. Qualification is in progress. Colocalization's ordinary 2.514475s +inclusive step cost inside 6.424050s execution is an upper bound, not measured +Costes-exclusive work or a promised saving. The separate 63 missing means +still require first-loss evidence. No numerical tolerance or parity gate changes. + +Compiler profiling confirms repeated effective-configuration construction and +cold schema analysis, but its timings overlap and are diagnostic only. A +private prototype is not admitted: default-factory/callback and mutable-state +laws remain unproved. This route is deferred while the larger unauthorized +metric execution is qualified. Full scaling and fresh figures remain pending; +the long-running optimization goal remains active. + +Requested CPA workspace production qualification +------------------------------------------------ + +Issue480 is integrated at ``38d02e20a795ad6f8c6457459c380995a798d0fe``. +The existing export settings retain authored workspace requests and panels; +nominal tool/axis leaves own the irreducible external grammar and derive +columns from the existing CPA dialect. The discarded settings validator is +removed. Publication uses the same existing export bundle/materialization +path. Full physical-file admission now consumes paths actually read by the +existing CPA equivalence report, instead of a manually synchronized suffix +roster. Unmatched files and failed readers cannot claim coverage. + +Original scoped R0 across all three roots and original R1 pass against2560; +138 agent controls and 81 root integration controls pass. An actual ordinary +QualityControl public run on clean, sealed38d observes 0.652361870s execution, +2.120112958s total and 1.088341713s compilation. Startup/readiness and shutdown +are excluded. No diagnostic hooks are installed. Both immutable saved native +repetitions pass database/schema/discrete/numerical measurements, nonempty +declared two-row Image table, requested workspace meaning, every physical +output and all source/input/dependency/native/controller freezes. This is a +correctness qualification, not a paired performance gain. The original V4 +missing-workspace inventory RED is retained unchanged. + +Complete receipt: +/var/tmp/openhcs-cpa-workspace-480-integrated-saved-native-v1-20261002.json +SHA256 72e22c3859081de0494ad24440d36a54b306adbaa509d8afcb9db87d671ece5d. +Ordinary source-freeze SHA256: +317de6735208ed9c99149d81296bf40dd3c1e70823e198da69b7cb7575d47e5d. +Standalone main isolation and its independent source checks are in progress; +the integrated result does not claim a main-only benchmark. + +The actual before-fix Beginner observation on frozen4904884 is execution +6.981277943s, total9.296563831s, compilation1.451889992s. It intentionally +preserves the reproduced issue487 metric bug and missing63parentmeans; no +parity exemption or speedup is claimed. Controller/source/environment/native +and complete actual input freezes pass. Source-freeze SHA256: +dc8a40519a10b7dd13ba9809a95bd6e2070622a326afa66ee2684adbe892a28d. +Issue487's source fix passes156 controls and original scoped R0/R1, but actual +candidate pipeline science and paired ordinary timing remain outstanding. + +Metric policy, saved parity and readiness follow-up, 2026-10-02 +------------------------------------------------------------- + +Main3705c071b13c29d354417d85dccf78c765b4a580 is normally merged at05ea538a. +Independent PR488 closes480 and PR489 closes487. Their production changes +are already integrated; subsequent main synchronization changes documentation +only. PR489 passes156 controls and original scoped R0/R1. Existing metric +declarations now derive enabled output columns; disabled Costes computations +and columns are omitted without changing enabled numerical implementations. + +Frozen baseline4904884 and sealed candidate d567d4d7 each have two cached +ordinary observations. Baseline execution5.527808428/5.327760458s and total +7.723725582/7.677667536s compare with candidate execution4.583703041/ +4.648376942s and total6.814110309/6.839743058s. Means improve execution +5.427784443 to4.616039992s (0.811744451s) and total7.700696559 to6.826926684s +(0.873769875s). Startup/readiness and shutdown are excluded; one worker +executes inline, with ordinary OUTCOMES and memory observation enabled. +This is a full-branch cached comparison, not an exclusive Costes attribution, +ABBA experiment, statistical claim or fully prewarmed qualification. + +The original baseline6.981277943/9.296563831s is excluded from that comparison: +three kernel overloads were written inside its pipeline timing window. The +later runs have no cache writes, which does not prove absence of cache loads. +Existing public library preparation omits object correlation and prepares +float64 threshold/RWC arrays; canonical runtime stages require float32 arrays. +A fresh private-cache public-preparation probe refuses later compilation and +cache loading: base passes; correlation, threshold and RWC fail. Requested +argument types exactly match retained production cache signatures. This is +readiness evidence using lawful synthetic values, not actual-input science +or performance evidence. Issue491 tracks canonical-stage preparation. + +All seven common CSV domains and two images are unchanged;100 unauthorized +Costes columns disappear (80 canonical and20 derived parent columns). +Candidate repeat scientific bytes are exact. Complete saved native replay +still fails solely for63 missing parent-mean features in both repetitions, +with no Costes, image, other numeric or physical-inventory differences. All +five relationship correlation keys and2093 pairs agree. Numerical tolerances +and required outputs are unchanged; full-case native parity remains RED. + +Issue492 identifies its first loss: all16 object-intensity tables exist, and +the producer has channel groups1/2/5/3, but compilation selects only child +channel3 for the prior-measurement input. Other channels never reach runtime +scope filtering. Repair belongs to the existing artifact relation/projection +contracts and must preserve child subject and image-set correlation. It adds +required scientific work; no speedup is claimed for this correctness repair. + +The bounded joint diagnostic pipeline succeeds with exact scientific bytes, +but capture admission is RED: no batch arrays are captured, and one final +store metadata event exceeds the4MiB cap. Earlier complete selection events +establish the bounded first-loss fact. The worker profile reaches28 serial +measurement invocations and zero batch calls; the earlier assumed batch +capture route is invalidated. A narrower serial-owner capture and nominal +batch-executor transport audit are next, not a claimed admitted replay. + +Cached comparison receipt: +/var/tmp/openhcs-beginner-metric-487-ordinary-cached-comparison-v1-20261002.json +SHA25636883d131c224f20b44139b02078c6ddba8b78a2e598901651910e12a92e4c2b. +Strict saved native receipt: +/var/tmp/openhcs-beginner-metric-487-strict-native-science-v1-20261002/receipt.json. +Fresh READY probe: +/var/tmp/openhcs-colocalization-public-ready-signatures-v2-20261002/observations.json. +Parent-mean first-loss receipt: +/var/tmp/openhcs-beginner-parent-means-first-loss-v2-20261002.json +SHA256c127ff3c79d82ddad4df38bb197606a25ee44a934e1909759922fc62b560f135. +Joint diagnostic admission/science receipt: +/var/tmp/issue487-joint-v2-original-admission-and-science-20261002.json +SHA256b29d5df0cd203e40a6e7f018e59b42d91e815a21af3e623811bc6b87a00ae53e. + +Whole-branch scalar-classifier R0 and issue479 validator-relocation R1 remain +RED. Installed419/433/435/450, original native image differences, full30-case +and scaling reruns and fresh figures remain open. The optimization goal stays +active; these measured gains do not redefine or complete the target. + +Canonical READY and current generic-query frontier +------------------------------------------------- + +PR493 is merged on mainb85f1e4add6075a71f4fd4e5ba7e2512e48990e5 and +closes491. Root normally integrates it atb015526c7c491abf80ea0958a08eb8723131c152. +Canonical image/object stages replace the duplicate kernel list and threshold +arithmetic.23 focused controls and original three-root R0/R1 pass; the actual +root fresh-cache integration control passes1test/9.86s. Compilation and disk +cache loading are refused after public preparation across IMAGE/OBJECT/BOTH +and threshold/Costes option states. This establishes readiness, not a speedup. + +One source-pinned Beginner diagnostic on clean b015 completes with exactly one +existing worker profile, nonempty existing runtime events,108 actual CP +executor calls,133 declared callable calls and no diagnostic errors. Profile +and event files occupy1.19MiB. Ten prelaunch controls preserve original calls, +errors, alias/mutation, descriptor/loader and thread partition behavior. The +instrumented12.9997s execution/15.2588s total is not an ordinary benchmark or +a regression. Profiling overhead is substantial and nonuniform. + +The same-thread orchestrator region partitions exactly12.988882s. This region +omits prior source/config decode; it is not the complete server-job clock. +Disjoint instrumented exclusive costs include module-output recording4.918576s, +declared callable invocations2.300491s, stack loading1.007764s, per-object output +recording0.920284s and worker residual0.861094s. cProfile costs overlap these +regions and must not be added or scaled to ordinary clocks. + +The determining generic query path makes312 full-column queries directly from +upstream parent-mean derivation:803488 structural-missing checks and195808 +qualification calls within those queries. Whole-worker structural checks are +1187866. Qualification precedes feature/source/row/object selection. Issue496 +tracks an operation-local schema/index fold on the existing nominal owners. +Opaque callback order, errors and mutation must remain observable; explicit +NaN propagation differs from default measurement-presence qualification. No new cache, +callback-identity dispatch or per-Relate numerical optimization is admitted. + +Strict replay of this diagnostic retains exactly63 missing parent means in both +saved native repetitions, with zero other numeric, image or inventory failures. +All seven CSVs/two TIFFs are byte-identical to ordinary candidated567; full +physical inventories agree and five correlation keys/2093pairs match. Source, +outputs, inputs, native and dependencies remain unchanged through replay. +Final all-zero science assertion correctly fails: no63-feature waiver exists. +Receipt /var/tmp/openhcs-current-generic-job-profile-strict-native-science-v1-20261002/receipt.json +SHA256bbc135e6de97628ddbabe8d350381a76a8af3000b4fba28f19e1879c27a220a8. + +Saved CSV readers cannot reconstruct preexport long rows, structural padding, +explicit missing cells, ownership or query order. That route is rejected; +metadata/relationship reader-domain errors are harness counterevidence, not +science failures. A bounded post492 actual logical-column/query capture must +demonstrate material absolute payoff before production optimization. The prior +ordinary Relate span near0.8s is only a whole-operation ceiling; the corrected +four-channel frontier and transfer remain unmeasured. + +Independent client residual evidence is0.818/0.804s for cached Beginner and +0.379s for QC. Two submissions occupy0.367/0.368s and0.120s respectively; +waiting beyond server clocks occupies0.03-0.08s. Four pipeline renders and +three config renders are observed in source, but their exact elapsed and byte +equality remain unproved. Blind memoization is rejected because shallow-frozen +documents contain mutable steps. This client-only route cannot remove server +execution work or independently close the seconds-wide target gap. + +Issue492's prepared four-channel/two-site repair passes its original controls +and guards. New prerequisite495 proves complete dynamic queries previously +ignored compiled producer path/backend, allowing foreign locations or an +ambiguous retained producer after legal rebinding. Existing planned query and +location authorities own the repair and point-in-time address law. Combined +qualification and actual full native63-field acceptance remain pending. + +The shared /home filesystem temporarily blocked Git commits. Only the owned +mutable worktree's website projection was made sparse through standard Git; +all75 website files were first preserved byte-exact at +/var/tmp/openhcs-owned-runtime-website-sparse-preserved-20261002/website with +their manifest receipt. Git objects retain the same files. This freed27MiB; +canonical ROOT, source code, benchmark inputs/outputs, dependencies and shared +caches were untouched. Earlier b015 source freezes completed before this +projection change; future freezes must explicitly record the sparse website. +All remaining acceptance and full-catalog/scaling/figure obligations stay open. + +Corrected producer domain, native parity and residual measurement frontier +------------------------------------------------------------------------ + +PR497 merges on main49f2fd3639363b7cc5e716eb3ba3a7358aa85429 and formally +closes492 and495. Its independently qualified production commit iseeed09233; +final6da19179 adds tracked qualification only. Original unchanged R0/R1 pass +against normally merged main6ce527a29 with no increased finding and unchanged +R1 budget.211 related controls pass;212 integrated controls pass11.78s with +two unrelated plugin configuration warnings. Global supplemental ownership +census covers704 modules/5228 original classes with no AST errors. + +InputObjectMeasurementSourceRelation derives complete measurement-producer +consumption through the existing input projection owner. The old planner and +RelateObjects selection algorithms are removed. Ordinary input and PLATE +selection remain intact. Dynamic queries resolve the actual compiled producer +path/backend and snapshot mutable group-address maps at query birth. The +meaningful old store-pickle control passed before planned lookup but failed +after caching a mapping-proxy address. Both lookup domains now use the existing +bounded store-owned cache lifetime; obsolete flat cache fields, initialization, +invalidation and transport paths are deleted. Ordered records, empty results, +replacement revision, eviction, primary/current/observation state and aliases +are covered. Arbitrary external plan/mapping callbacks and query-target callback +counts under bounded eviction remain outside the concrete production audit. + +Root integrates production at5ed0d13978db1cda92a490614d326525ad552af7. A fresh +uninstrumented ordinary Beginner1w1t completes with execution4.650205374s, +compilation1.603918076s and total7.070084621s. Mandatory READY, registry/kernel +warmup, process startup and shutdown are excluded. Execution is inline with one +thread; no measured fork/spawn startup. Default OUTCOMES and ordinary memory +observation are retained. The tracked75-file website-only sparse projection is +explicitly inventoried; no production source/input/dependency is omitted. +Source, Python, installed dependencies, native binaries and exact image/CPPipe +hashes are fixed before and after. This single correctness observation is not +causal optimization or statistical evidence. + +The unchanged strict reader consumes all seven CSVs and two TIFFs against BOTH +retained native repetitions: ZERO numeric/measurement differences, ZERO image +differences, complete physical inventory, and five correlation keys/2093pairs +on both sides. The63 missing parent means are restored. Only MyExpt_Nuclei.csv +changes scientifically versus the previous ordinary result. Reader/source and +output hashes remain unchanged through comparison. This execution uses the +integrated394/479 reader: it qualifies the producer repair's outputs, not a +standalone main-only execution or whole394 architecture gate. +Receipt /var/tmp/openhcs-child-measurements-492-integrated-strict-native-science-v1-20261002/receipt.json +SHA25637930ee4622c5fc1e067fc853bef1ae627ebe74546c27c3cc65d9fb476343ba0. +Tracked independent qualification: +docs/validation/child-measurement-producer-selection-492-495-20261002/QUALIFICATION.rst. + +The retained native warm-process invocations are15.270141648s and14.634740414s, +mean14.952441031s. A descriptive comparison to this single corrected execution +is3.215x execution and2.115x including OpenHCS compilation/client completion; +these are different sample counts, not a causal change or statistical claim. +Native invocation includes prepare/modules/post-run/close, with import/JVM, +CPPipe load and its warmup excluded. Startup is not added to either pipeline. +Other cases and full-catalog/scaling figures are not qualified by this case. + +The next actual logical-query capture executes438 queries over seven tables, +not four times the previous312. Its processor succeeds and all nine scientific +files are byte-identical to the qualified5ed ordinary output; complete physical +inventories match. Diagnostic admission correctly fails before writing any +logical graph: unchanged V6 refuses the150000-node unique-instance inventory. +No partial graph, increased limit, saved-CSV substitution or performance claim +is admitted. The profiled/captured14.1409s execution is diagnostic overhead, +not an ordinary regression. Preserve the original failure and source freeze. +Artifact /var/tmp/openhcs-measurement-query-logical-capture-v1-20261002/observations.json. +Scientific-byte receipt: +/var/tmp/openhcs-measurement-query-logical-capture-v1-scientific-byte-comparison-20261002.json. + +A revised diagnostic representation may pack numeric answer-oracle leaves while +preserving every actual query and the complete determining input/schema/alias +graph. Node/category counts and original replay must establish admission before +any matched candidate timing or production change. Existing150000-node, +32MiB graph,4MiB ledger and512MiB free-space limits remain unchanged. Pure +callback declarations alone do not license skipping arbitrary scalar equality +or string effects; original opaque callback/error/mutation order remains a +separate production wiring gate. + +The output-recording ownership audit also rejects a misleading second frontier: +coarse recorder-exclusive4.9186s contains the query496 subtree. In cProfile, +record_module_outputs self time is0.000638s, and measurement-table construction +contains Relate augmentation4.1412s/upstream query3.3457s. Those diagnostic +times cannot be added. Independent100-call object recording is dominated by +split_scoped_rows0.5620s diagnostic. Its18735-row domain has80 colocalization, +16 concatenated and4 shape batches. Existing row owners could remove dictionary +expansion, but no credible multi-second ordinary payoff is established; the +standalone route is rejected pending a necessary shared-query dependency. +Audit /var/tmp/openhcs-b015-output-recording-ownership-audit-20261002.json +SHA256b0bf2c90f50c1da5022f6aa2a657779f0bdd9390c0adfb022a86b5db21b10779. + +Root normally merges current main after all5ed jobs complete, at899aaddce; +that merge changes documentation/evidence only relative to5ed production. +Whole394 scalar-classifier R0 and479 validator-relocation R1 remain RED. +Installed419/433/435/450, outstanding native image differences, full30/scaling +reruns and fresh figures remain open. The unlimited optimization goal is ACTIVE. + +Profiler-free exclusive ledger and plate export correction +--------------------------------------------------------- + +The source-pinned5d50a0bb46ab4ccdd17bb9924ce698893f5ed85b diagnostic completes +with BOTH cProfile/runtime profiler disabled,24 coarse nominal boundaries and +14 meaningful prelaunch controls. Source/environment/native/input and explicit +website-only sparse witnesses remain fixed before/after. It retains32,223bytes +of JSON, no profile/capture artifacts and no diagnostic errors. All nine +scientific output bytes and the complete11-file inventory are identical to the +strict-both-native-qualified5ed ordinary result. Managed metadata differences +are recorded separately. No new native execution or tolerance waiver is used. + +Public execution4.703251600s/compilation1.568818331s/total7.089778207s are +diagnostic, not a causal gain or an estimate of wrapper overhead. The execution +region4.692535908s partitions into23 nonzero owner groups:133 hooked callable +invocations1.923475937s and2.769059971s outside that boundary. The invocation +boundary includes metadata/filtering and does NOT cover all registered calls. +In particular, execute_plate_scoped_steps directly resolves/calls PLATE exports. +Actual ExportToSpreadsheet progress is0.707153320s; it lies inside the +server-exclusive0.734300637s bucket. These intervals overlap and cannot be +added or subtracted as exact disjoint method costs. Server reconstruction is +therefore not established as the dominant0.734s term. Source/config decoding +also precedes this region. Compiler-exclusive1.009028861s includes genuine +fresh compilation, not a second artifact-reuse compile. + +Current measured exclusive query cost is0.481040613s/438calls. Nonquery module +recording is0.405474477s (inclusive0.886515090s), object recording0.156714182s, +stack loading0.184871211s, CP contract execution0.212007483s, worker remainder +0.140013289s, CP module remainder0.130000117s and image request0.086237226s. +Saving/publication/unstacking individually are smaller. Recording remainder +contains actual table creation, parent means and other required processing; it +is not isolated provenance. All-query/all-module/all-object recording has an +impossible elimination ceiling1.043229273s before retaining mandatory work. + +This invalidates the single-query route as the main multi-second fix. Its +18-control compact numeric-oracle capture preparation is retained but will not +be frozen/launched/polished without a necessary broader dependency. Existing +V6 and original limits remain unchanged. The new investigation maps common +columnar table/schema derivation across queries, recorders and the actual +spreadsheet publication path, retaining opaque scalar/callback semantics. +The exporter needs its own authority/phase analysis before a payoff claim. + +Ledger /var/tmp/openhcs-current-generic-runtime-ledger-v2-20261002/observations.json. +Corrected analysis /var/tmp/openhcs-current-generic-runtime-ledger-v2-analysis-plate-corrected-20261002.json +SHA2568b74fb088fae2527563fb42d38f3dedf5ab5ed936c5de805cd394343e3d0529c. +Byte gate /var/tmp/openhcs-current-generic-runtime-ledger-v2-scientific-byte-gate-20261002.json. +Original first-classification and correction receipts are both retained. +PR394's current description is rewritten around the remaining implementation; +merged fixes are described as dependencies and closed issue clauses removed. +Open persisted-publication134 now has an explicit closing link to draft394, +alongside384/419/433/435/450/479/176/496. None is claimed complete merely linked. diff --git a/docs/validation/singleton-label-invocation-477-20261002.rst b/docs/validation/singleton-label-invocation-477-20261002.rst new file mode 100644 index 0000000000..faad1c18a4 --- /dev/null +++ b/docs/validation/singleton-label-invocation-477-20261002.rst @@ -0,0 +1,92 @@ +Singleton label projection at the final invocation boundary (#477) +================================================================= + +The ordinary translocation benchmark failed on frozen ``6e69b851`` before +producing a successful well. Its OverlayOutlines invocation paired a scalar +640-by-640 RGB image with object labels on a declared singleton runtime root. +The image had a channel axis but no image-plane projection. Treating the RGB +channels as image planes would violate the existing strict axis contract. + +The input binding request initially derived label shape with the callable's +FLEXIBLE default. The module executor resolved the final NATURAL mode later, +after binding, so its semantic controls and the bound labels could disagree. + +Existing nominal ownership +------------------------- + +``ObjectLabelInputExecutionMode.invocation_kwargs`` now owns this final +projection. MATCH_IMAGE_STACK with a NATURAL scalar image may project an +already-bound label argument only when the original runtime declaration proves +exactly one plane. It delegates to the existing ``RuntimeSliceProjection`` and +merges the resolved semantic controls once. The module executor delegates to +that owner after resolving execution mode. + +Explicit FULL_STACK label declarations and FULL_STACK execution remain intact. +Multiple or unproved runtime axes and retained image-plane projections keep +their original strict guards. The fix neither invents an image axis nor reloads +artifacts. Redundant intermediate image/default-mode locals are removed. No +new policy class, cache, per-module exception, or color-processing change is +introduced. + +Source and local validation +--------------------------- + +The final isolated source is ``ccaa4cba6a1eca818cd544438941f1c1cedb3e32``, +integrated into PR #394 by ``1763ef07eba959dbefa7643cb90c8230fdf12518``. +The initial inline implementation and its original R0 failure are retained in +history; that route added GodClass, boolean-chain and foreign-absence metrics. +The final existing-owner implementation passes unchanged original R0 metrics +and original R1 with its original 160-second budget. + +The original reproducer recorded two expected failures and five passing +controls. The repaired module-consumer gate passes 402 controls. The global +source census parsed 703 modules with no errors. All five registered +MATCH_IMAGE_STACK consumers currently declare FLEXIBLE processing contracts. +External custom MATCH consumers with a non-FLEXIBLE stack-only contract have +not been qualified; this report makes no blanket callback-equivalence claim. + +The source-qualified receipt is +``/var/tmp/issue477-policy-owner-qualified-receipt-20261002.json`` with SHA256 +``7ebcd89739bbe58cf7630c093f02b92ecf41732a196073641ec373e4d9b019ca``. + +Actual public pipeline consumer +------------------------------- + +The frozen integrated source at +``/var/tmp/openhcs-pr394-singleton-label-qualified-20261002`` remains immutable. +Its ordinary public run completed both translocation and WoundHealing with one +successful well each, a reused READY server, one inline worker and one thread +on CPU 5. All input, native-binary, environment, dependency and tracked-source +freezes pass. Server/library/kernel preparation and shutdown are outside the +pipeline clocks; compilation and OUTCOMES closure remain inside total. + +The single observations are: + +================= ============== ============== ============== +Case Compilation(s) Execution(s) Total(s) +================= ============== ============== ============== +Translocation 0.527004 1.570796 2.539227 +WoundHealing 1.296431 2.897727 4.421997 +================= ============== ============== ============== + +These are correctness observations, not matched optimization evidence. +Wound's higher compilation observation is retained and its cause is unknown. +The original failed translocation clocks remain excluded from speedup claims. +The observation receipt is +``/var/tmp/openhcs-pr394-singleton-label-ordinary-v1-20261002/observations.json``; +its source-freeze SHA256 is +``ea4969dbfe48e7a42ff039b07a32eb0d2464374362b89588682483fa33039128``. + +Wound's complete authored output inventory, table comparison and exact +scientific bytes pass against the qualified ``9aa3918d`` reference. Its +two-row Image.csv SHA256 remains +``939735fa17a067137fea2adc77d57f9ce1df24bfcb6588c49023de1cc3998677``. +The read-only scientific receipt is +``/var/tmp/openhcs-pr394-singleton-label-wound-science-20261002.json`` with SHA256 +``7a8c3cafa33e54394d9a24f7744c6a6b641e48c6f712f1baab7c632dcace0655``. + +Translocation still requires fresh warm-native execution and complete authored +table, image, discrete-value and inventory comparison before issue acceptance. +The first native-controller preparation correctly rejected an auxiliary CSV +misclassified as an image input; its failure is retained. No tolerance change, +scientific-output exclusion or issue-closure claim is made here. diff --git a/docs/validation/synthetic_live_fractional_z_points.py b/docs/validation/synthetic_live_fractional_z_points.py new file mode 100644 index 0000000000..458a521aad --- /dev/null +++ b/docs/validation/synthetic_live_fractional_z_points.py @@ -0,0 +1,106 @@ +"""Tiny original PURE_3D producer -> typed Points writer -> live-domain control.""" +from pathlib import Path +import argparse +from openhcs.serialization.json import to_jsonable +import json +import numpy as np +import tifffile + +from openhcs.core.config import NapariStreamingConfig +from openhcs.core.memory import numpy as numpy_func +from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.core.measurement_row_materialization import MeasurementSparseColumnarRows +from openhcs.core.runtime_measurements import MeasurementTable, MeasurementSubject, MeasurementScope, ObjectCoreMeasurementFeature +from openhcs.core.runtime_tabular_values import FieldSpec +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.processing.materialization import MaterializationSpec, PointROIOptions, materialize +from openhcs.core.steps.function_artifact_materialization import ArtifactStreamSourceMetadataAuthority +from openhcs.core.steps.stream_component_semantics import StreamComponentMessageExtraAuthority, StreamSourceComponentMetadataItems +from openhcs.core.streaming_config_factory import StreamingViewerSurface, StreamingViewerRuntimeConfig +from openhcs.core.streaming_config_declarations import ViewerType +from polystore.streaming.viewer_transport import ViewerStreamSourceIdentity +from polystore.filemanager import FileManager +from polystore.disk import DiskStorageBackend +from polystore.roi import load_rois_from_zip +from openhcs.core.roi_point_metadata import ROIFractionalZ +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from zmqruntime.viewer_protocol import ViewerTransportEndpoint +from zmqruntime.config import TransportMode + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output', type=Path, required=True, help='New owned persistent synthetic directory; must not exist') +root = parser.parse_args().output +root.mkdir(exist_ok=False) +components = tuple({'well':'A01', 'site':1, 'channel':1, 'z_index':z, 'timepoint':1} for z in range(4)) +paths = tuple(str(root / f'A01_s001_w1_z{z:03d}_t001.tif') for z in range(4)) +volume = np.zeros((4,5,7), dtype=np.uint16) +volume[1:3,1:3,2:4] = 1 +for z, path in enumerate(paths): + tifffile.imwrite(path, volume[z]) +provenance = SourceImageProvenancePlanes.from_components(paths=paths, component_metadata=components) +metadata = ImagePayloadMetadata(source_image_provenance_planes=provenance, source_component_metadata={k:v for k,v in components[0].items() if k!='z_index'}, source_voxel_spacing=SourceVoxelSpacing((2.0,0.65,0.65)),plane_axis=RuntimePlaneAxis.SOURCE_BINDING) + +@numpy_func(contract=ProcessingContract.PURE_3D) +def synthetic_centres(image: np.ndarray) -> MeasurementTable: + assert image.shape == (4,5,7) + z,y,x = np.argwhere(image > 0).mean(axis=0) + return MeasurementTable(name='synthetic_centres', + rows=MeasurementSparseColumnarRows.from_rows(({'object_label':7,'center_z':float(z),'center_y':float(y),'center_x':float(x)},), + fields=(FieldSpec('object_label',int),FieldSpec('center_z',float),FieldSpec('center_y',float),FieldSpec('center_x',float))), + source_image_provenance_planes=provenance, source_component_metadata=metadata.source_component_metadata, + subject=MeasurementSubject(MeasurementScope.OBJECT,'synthetic','object_label')) + +table = synthetic_centres(metadata.attach_to(volume)) +spec = MaterializationSpec(PointROIOptions(z_feature=ObjectCoreMeasurementFeature.CENTER_Z,y_feature=ObjectCoreMeasurementFeature.CENTER_Y,x_feature=ObjectCoreMeasurementFeature.CENTER_X)) +archive = materialize(spec,data=table,path=str(root/'centres'),filemanager=FileManager({'disk':DiskStorageBackend()}),backends=['disk'],backend_kwargs={}) +items = ArtifactStreamSourceMetadataAuthority.metadata_items(materialization_spec=spec,data=table,fallback_source_identity=None) +surface = StreamingViewerSurface( + runtime_config=StreamingViewerRuntimeConfig(transport_endpoint=ViewerTransportEndpoint(host='127.0.0.1',port=6199,transport_mode=TransportMode.TCP),persistent=True,viewer_type=ViewerType.NAPARI), + display_config=NapariStreamingConfig(enabled=True), source=ViewerStreamSourceIdentity(microscope_handler=None,plate_path=None)) +authority = StreamComponentMessageExtraAuthority.from_viewer_surface(surface,source_metadata_items=items) +saved_rois = load_rois_from_zip(Path(archive)) +saved_metadata = ROIArchiveSourceMetadata.decode(saved_rois) +saved_domain = ROIFractionalZ.source_component_domain(saved_rois, saved_metadata) +reopened = StreamComponentMessageExtraAuthority.from_viewer_surface(surface, source_metadata_items=StreamSourceComponentMetadataItems.from_values(saved_domain)) +result = dict(saved_domain=saved_domain, archive_scalar_metadata=saved_metadata.source_component_metadata, archive=archive,shape=volume.shape,rows=tuple(table.iter_row_mappings()),source_component_order=authority.layout.component_order,declared_domain=authority.payload().to_wire_mapping(), scientific_data_opened=False,runtime_started=False) +print(json.dumps(to_jsonable(result),sort_keys=True)) +assert 'z_index' not in authority.layout.component_order, 'Expected original live-source defect no longer reproduces; update this engineering control' +metadata = saved_metadata +rois = saved_rois +live = authority +archive = Path(archive) + +from polystore.roi_converters import NapariROIConverter +from polystore.streaming.identity import StreamProducerIdentity +from polystore.streaming_constants import StreamingDataType +from openhcs.runtime.napari_streaming_handlers import NapariStreamLayerItem, NapariStreamLayerAddress +from openhcs.runtime.viewer_component_system import ViewerComponentValueDomainPayload +from openhcs.runtime.napari_viewer_server import NapariPointsLayerDisplayHandler + +anchor = metadata.source_provenance.for_source_plane(0).scalar_source_identity +domain = ViewerComponentValueDomainPayload.from_wire_mapping(reopened.payload().to_wire_mapping()['component_value_domain'],context='own synthetic source-bearing points') +item = NapariStreamLayerItem( + data=NapariROIConverter.rois_to_shapes(rois), + producer=StreamProducerIdentity.pipeline_output(output_kind='artifact',output_key='synthetic_centres',projection_key='synthetic_centres',step_name='Own engineering only',pipeline_position=0,artifact_kind='roi'), + address=NapariStreamLayerAddress(components=anchor.component_metadata,path=str(archive),stream_layer_data_type=StreamingDataType.POINTS), + image_metadata=metadata,plane_component_domain=domain) +handler = NapariPointsLayerDisplayHandler() +results = {} +for name, fallback in [('none',None),('common',metadata.source_provenance.scalar_source_identity),('first_plane',anchor)]: + items = ArtifactStreamSourceMetadataAuthority.metadata_items(materialization_spec=spec,data=table,fallback_source_identity=fallback) + authority = StreamComponentMessageExtraAuthority.from_viewer_surface(surface,source_metadata_items=items) + try: + geometry = handler.geometric_component_values([item],authority.component_axis_semantics) + except ValueError as error: + results[name] = dict(order=authority.layout.component_order,error=str(error)) + else: + raise AssertionError(f'Incomplete live domain unexpectedly accepted: {name}: {geometry!r}') +positive = handler.geometric_component_values([item],reopened.component_axis_semantics) +assert results['none']['error'] == 'Fractional-Z point ROI requires a projected z_index axis.' +assert results['common']['error'] == results['none']['error'] +assert 'exceeds the declared Z domain' in results['first_plane']['error'] +assert positive == {'z_index':[0,1,2]} +print(json.dumps(dict(negative=results,original_domain_positive=positive,viewer_started=False,connected=False),sort_keys=True)) diff --git a/openhcs/core/aligned_image_payload.py b/openhcs/core/aligned_image_payload.py index 7c6dfae012..c939865b21 100644 --- a/openhcs/core/aligned_image_payload.py +++ b/openhcs/core/aligned_image_payload.py @@ -423,6 +423,26 @@ def composition_payloads(self) -> tuple[Any, ...]: ... @abstractmethod def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: ... + @staticmethod + def copy_whole_image( + value: RuntimeArrayData, + *, + memory_type: str, + device_id: int | None, + ) -> RuntimeArrayData: + """Copy pixels, mask and metadata without adding a composition axis.""" + copied_data = stack_runtime_slices( + (image_payload_data(value),), memory_type, device_id, + )[0] + mask = image_payload_mask(value) + copied_mask = ( + None if mask is None + else stack_runtime_slices((mask,), memory_type, device_id)[0] + ) + return image_payload_metadata(value).replace_fields().payload_with( + copied_data, copied_mask, + ) + def composition_source_metadata(self) -> tuple[ImagePayloadMetadata, ...]: return tuple( self.composition_payload_metadata(image_payload_metadata(payload)) @@ -992,6 +1012,11 @@ class AlignedImageSliceContext: projection_key: str artifact_kind: str | None = None + @property + def persisted_source_alias(self) -> str | None: + """Expose a declared artifact name, preserving anonymous main flow.""" + return self.output_key if self.artifact_kind is not None else None + @classmethod def main_flow( cls, @@ -1140,6 +1165,17 @@ def composition_payloads(self) -> tuple[Any, ...]: def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: return ImagePayloadMetadataCompositionMode.STACK + @property + def projected_output_composition_mode(self) -> ImagePayloadMetadataCompositionMode | None: + """Declare the outer runtime axis retained by projected output members.""" + return self.composition_metadata_mode + + def plane_axis_for_output_context( + self, context: AlignedImageSliceContext, + ) -> RuntimePlaneAxis | None: + """An explicitly aligned stack declares its outer runtime-slice domain.""" + return RuntimePlaneAxis.RUNTIME_SLICE + def composition_payload_metadata( self, metadata: ImagePayloadMetadata ) -> ImagePayloadMetadata: @@ -1194,6 +1230,58 @@ def projected_output_slices( for output_slice in payload_slices_for_alignment(payload): yield output_slice, context + def output_values_for_artifact_specs( + self, + canonical_specs: tuple[ArtifactSpec, ...], + ) -> dict[ArtifactSpecRef, Any]: + """Bind one complete declared main-flow roster to exact slice contexts.""" + + if not self.slice_contexts: + raise ValueError( + "Multiple canonical output specs require exact AlignedImageStack " + "slice contexts; positional slice order is not artifact identity." + ) + + specs_by_context = { + (spec.name, spec.artifact_type.value): spec for spec in canonical_specs + } + if len(specs_by_context) != len(canonical_specs): + raise ValueError("Canonical output ABI contains duplicate named contexts.") + resolved: dict[ArtifactSpecRef, Any] = {} + for payload, context in zip( + self.slices, + self.slice_contexts, + strict=True, + ): + if context.output_kind != AlignedImageSliceContext.MAIN_FLOW_OUTPUT_KIND: + raise ValueError( + "Canonical AlignedImageStack contains a non-main-flow slice " + f"context: {context!r}." + ) + context_key = (context.output_key, context.artifact_kind) + spec = specs_by_context.get(context_key) + if spec is None: + raise ValueError( + "Canonical AlignedImageStack context is not declared by the " + f"callable ABI: {context!r}." + ) + ref = spec.ref() + if ref in resolved: + raise ValueError( + "Canonical AlignedImageStack contains duplicate context for " + f"{ref!r}." + ) + resolved[ref] = payload + missing = tuple( + spec.ref() for spec in canonical_specs if spec.ref() not in resolved + ) + if missing: + raise ValueError( + "Canonical AlignedImageStack does not carry every declared output: " + f"{missing!r}." + ) + return resolved + def output_payload( self, artifact_ref: ArtifactSpecRef, @@ -1234,6 +1322,33 @@ def composition_payloads(self) -> tuple[Any, ...]: def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: return ImagePayloadMetadataCompositionMode.BUNDLE + @property + def projected_output_composition_mode(self) -> ImagePayloadMetadataCompositionMode | None: + """Flatten declared inner runtime planes, retaining other named image domains.""" + if any( + image_payload_metadata(payload).plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + for payload in self.slices + ): + return ImagePayloadMetadataCompositionMode.STACK + if len(self.slices) == 1: + return None + return self.composition_metadata_mode + + def plane_axis_for_output_context( + self, context: AlignedImageSliceContext, + ) -> RuntimePlaneAxis | None: + """Resolve a named output's original inner domain before leaf projection.""" + payloads = tuple( + payload for payload, declared_context in zip(self.slices, self.slice_contexts, strict=True) + if declared_context == context + ) + if len(payloads) != 1: + raise ValueError( + "Named image output context requires exactly one original payload: " + f"{context!r}; found {len(payloads)}." + ) + return image_payload_metadata(payloads[0]).plane_axis + def composition_payload_metadata( self, metadata: ImagePayloadMetadata ) -> ImagePayloadMetadata: diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index 635e6f958a..2d11eee696 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -24,6 +24,7 @@ from openhcs.core.component_set import ComponentSet if TYPE_CHECKING: + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity from openhcs.core.callable_contract import FunctionStepExecutionScope from openhcs.core.function_patterns import FunctionInvocationKey from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy @@ -215,6 +216,11 @@ def materialization_base_path( del source_filename, uses_source_identity_filename, image_output_dir return Path(analysis_output_dir) / descriptor_filename + @classmethod + def retained_filename_qualifier(cls, artifact_name: str) -> str | None: + """Retain families whose existing paths already own artifact identity.""" + return None + @classmethod def normalize_runtime_payload( cls, @@ -385,6 +391,11 @@ class ImageArtifactType(ArtifactType): participates_in_main_flow_output = True carries_source_image_context = True + @classmethod + def retained_filename_qualifier(cls, artifact_name: str) -> str: + """Distinguish named retained images sharing one physical source plane.""" + return artifact_name + @classmethod def projected_materialization_base_path( cls, @@ -1054,6 +1065,19 @@ def participates_in_persistent_materialization(self) -> bool: def uses_source_identity_filename(self) -> bool: """Return whether this materialization names files by source identity.""" + @abstractmethod + def filename_qualifier(self, output_plan: ArtifactOutputPlan | None) -> str | None: + """Derive the output role required by this materialization purpose.""" + + def filename_identity_for_output( + self, + identity: FunctionOutputIdentity, + output_plan: ArtifactOutputPlan | None, + ) -> FunctionOutputIdentity: + """Apply a declared role through the shared filename identity algorithm.""" + qualifier = self.filename_qualifier(output_plan) + return identity if qualifier is None else identity.with_filename_qualifier(qualifier) + def _coerce_artifact_plan_type( plan_type: type["ArtifactPlan"], @@ -2153,6 +2177,37 @@ def __iter__(self): def __getitem__(self, index): return self.specs[index] + def stack_broadcast_source_indices(self) -> tuple[int | None, ...]: + """Resolve exact stack-broadcast owners from declared input relations.""" + + indices_by_ref: dict[ArtifactSpecRef, list[int]] = {} + for input_index, spec in enumerate(self.specs): + indices_by_ref.setdefault(spec.ref(), []).append(input_index) + + result: list[int | None] = [] + for input_index, spec in enumerate(self.specs): + sources = spec.stack_broadcast_sources() + if len(sources) > 1: + raise ValueError( + f"Input {spec.ref()!r} declares multiple stack-broadcast " + f"owners: {sources!r}." + ) + if not sources: + result.append(None) + continue + source_indices = tuple(indices_by_ref.get(sources[0], ())) + if len(source_indices) != 1: + raise ValueError( + f"Input {spec.ref()!r} requires exactly one active occurrence " + f"of stack-broadcast owner {sources[0]!r}, got " + f"{source_indices!r}." + ) + source_index = source_indices[0] + if source_index == input_index: + raise ValueError(f"Input {spec.ref()!r} cannot broadcast from itself.") + result.append(source_index) + return tuple(result) + def of_artifact_type( self, artifact_type: ArtifactTypeValue, diff --git a/openhcs/core/callable_contract.py b/openhcs/core/callable_contract.py index 701a3dda0e..18e75a7f7f 100644 --- a/openhcs/core/callable_contract.py +++ b/openhcs/core/callable_contract.py @@ -22,6 +22,7 @@ Mapping, TypeVar, cast, + get_args, get_type_hints, overload, ) @@ -33,6 +34,7 @@ coerce_enum_member, declared_enum_type, enum_member_type, + is_union_type, resolve_annotated, validate_annotation_value, ) @@ -341,9 +343,152 @@ class CallableMetadata: image_payload_consumption: ImagePayloadConsumption = ImagePayloadConsumption.NATURAL request_binding: "CallableRequestBinding | None" = None prepare: Callable[..., object] | None = None + canonical_signature: inspect.Signature | None = None + raw_runtime_signature: inspect.Signature | None = None primary_image_carrier_requirement: PrimaryImageCarrierRequirement | None = None primary_image_carrier_transition: PrimaryImageCarrierTransition | None = None + @staticmethod + def prepared_callable_signature(func: Callable[..., object]) -> inspect.Signature | None: + """Admit an owned snapshot only for its exact semantic callable boundary.""" + raw = getattr(func, FunctionContractAttribute.raw_processing_function, None) + if raw is not None and raw is not func: + return None + signature = getattr(func, FunctionContractAttribute.canonical_signature, None) + if signature is not None and not isinstance(signature, inspect.Signature): + raise TypeError("Prepared callable signature must be an inspect.Signature or None.") + return signature + + @classmethod + def callable_signature(cls, func: Callable[..., object]) -> inspect.Signature: + """Read the prepared exact ABI; unprepared wrapper/authoring views stay live.""" + signature = cls.prepared_callable_signature(func) + return inspect.signature(func) if signature is None else signature + + def resolve_canonical_raw_callable(self, func: Any) -> Callable[..., object]: + """Return the declaration-owned callable whose signature defines behavior.""" + + declared = self.raw_processing_function + if declared is None: + declared = func + return _resolve_declared_callable(declared) + + def resolve_raw_runtime_callable(self, func: Any) -> Callable[..., object]: + """Remove runtime wrappers without crossing a semantic request binding.""" + + canonical = self.resolve_canonical_raw_callable(func) + if self.request_binding is None: + return inspect.unwrap(canonical) + + binding_key = FunctionContractAttribute.callable_request_binding + + def is_request_boundary(candidate: Callable[..., object]) -> bool: + namespace = _callable_namespace(candidate) + if binding_key not in namespace: + return False + wrapped = namespace.get("__wrapped__") + return not callable(wrapped) or binding_key not in _callable_namespace( + wrapped + ) + + resolved = inspect.unwrap(canonical, stop=is_request_boundary) + if binding_key not in _callable_namespace(resolved): + raise RuntimeError( + f"Callable {CallableProjection.from_callable(func).name!r} declares a request binding, but " + "its canonical raw callable has no request-binding boundary." + ) + return resolved + + def resolve_signature(self, raw_callable: Callable[..., object]) -> inspect.Signature: + """Resolve the semantic ABI once after declaration preparation.""" + signature = inspect.signature(raw_callable) + annotations = ( + self.request_binding.public_annotations_dict + if self.request_binding is not None + else get_type_hints(raw_callable, include_extras=True) + ) + return signature.replace( + parameters=tuple( + parameter.replace(annotation=annotations.get(name, parameter.annotation)) + for name, parameter in signature.parameters.items() + ), + return_annotation=annotations.get("return", signature.return_annotation), + ) + + @staticmethod + def primary_input_name(parameters: Mapping[str, inspect.Parameter]) -> str | None: + for parameter in parameters.values(): + if parameter.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ): + return parameter.name + return None + + @classmethod + def _nominal_argument_types(cls, annotation: object) -> tuple[type, ...]: + annotation = resolve_annotated(annotation) + if is_union_type(annotation): + return tuple( + argument_type for member in get_args(annotation) + for argument_type in cls._nominal_argument_types(member) + ) + return ( + (annotation,) + if annotation is not Any and isinstance(annotation, type) + else () + ) + + def raw_main_flow_call_argument(self, source_payload: Any, signature: inspect.Signature) -> Any: + """Admit a declared nominal carrier at the canonical image boundary.""" + from openhcs.core.runtime_image_values import image_payload_data + + primary_name = self.primary_input_name(signature.parameters) + annotation = None if primary_name is None else signature.parameters[primary_name].annotation + if ( + annotation is inspect.Parameter.empty + and self.image_payload_consumption is ImagePayloadConsumption.COMPOSED + ): + return source_payload + return source_payload if any( + isinstance(source_payload, argument_type) + for argument_type in self._nominal_argument_types(annotation) + ) else image_payload_data(source_payload) + + def canonical_signature_for(self, func: Any) -> inspect.Signature: + """Use the prepared semantic ABI; unprepared authoring remains live.""" + if self.canonical_signature is not None: + return self.canonical_signature + return self.resolve_signature(self.resolve_canonical_raw_callable(func)) + + def raw_runtime_signature_for(self, func: Any) -> inspect.Signature: + """Own the distinct raw ABI or the shared canonical target snapshot.""" + if self.raw_runtime_signature is not None: + return self.raw_runtime_signature + if self.canonical_signature is not None: + return self.canonical_signature + return self.resolve_signature(self.resolve_raw_runtime_callable(func)) + + def with_prepared_signatures(self, canonical: Callable, runtime: Callable) -> "CallableMetadata": + """Snapshot each actual target independently after declaration preparation.""" + return dataclasses.replace( + self, canonical_signature=self.resolve_signature(canonical), + raw_runtime_signature=None if runtime is canonical else self.resolve_signature(runtime), + ) + + def with_missing_signatures(self, canonical: Callable[[], Callable], runtime: Callable[[], Callable]) -> "CallableMetadata": + """Reuse library readiness or capture the newly prepared authored targets.""" + if self.canonical_signature is not None: + return self + return self.with_prepared_signatures(canonical(), runtime()) + + def publish_signatures(self, namespace: MutableMapping[str, Any]) -> None: + """Replace the namespace pair as one library preparation epoch.""" + namespace[FunctionContractAttribute.canonical_signature] = self.canonical_signature + namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) + if self.raw_runtime_signature is not None: + namespace[FunctionContractAttribute.raw_runtime_signature] = self.raw_runtime_signature + @property def artifact_output_policy(self) -> type[ArtifactOutputPolicy]: """Project recording ownership from this callable's adapter declaration.""" @@ -427,13 +572,14 @@ def from_projection( runtime_adapter = runtime_adapter_spec_from_callable(projection.func) if runtime_adapter is not None and callable(projection.func): runtime_adapter.validate_callable_signature(projection.func) + declared_artifact_inputs = _artifact_specs_from_namespace( + namespace, projection.name, FunctionContractAttribute.artifact_inputs, + ) + signature = ( + cls.callable_signature(projection.func) if callable(projection.func) else None + ) artifact_inputs = _artifact_input_specs_from_projection( - projection, - _artifact_specs_from_namespace( - namespace, - projection.name, - FunctionContractAttribute.artifact_inputs, - ), + projection, declared_artifact_inputs, signature, ) return cls( input_memory_type=reader.optional_string( @@ -450,6 +596,7 @@ def from_projection( projection, reader, artifact_inputs, + signature, ), artifact_outputs=_artifact_specs_from_namespace( namespace, @@ -471,7 +618,7 @@ def from_projection( FunctionContractAttribute.allowed_group_by, ), runtime_adapter=runtime_adapter, - runtime_context_parameter=_runtime_context_parameter(projection, reader), + runtime_context_parameter=_runtime_context_parameter(projection, reader, signature), execution_scope=reader.optional_execution_scope( FunctionContractAttribute.execution_scope, ), @@ -506,6 +653,12 @@ def from_projection( prepare=reader.optional_callable( FunctionContractAttribute.processing_prepare ), + canonical_signature=reader.optional_signature( + FunctionContractAttribute.canonical_signature, + ), + raw_runtime_signature=reader.optional_signature( + FunctionContractAttribute.raw_runtime_signature, + ), ) def without_prepare(self) -> "CallableMetadata": @@ -525,6 +678,10 @@ def with_raw_processing_function( def as_namespace(self) -> dict[str, object]: """Project typed metadata into callable declaration keys.""" namespace: dict[str, object] = {} + if self.canonical_signature is not None: + namespace[FunctionContractAttribute.canonical_signature] = self.canonical_signature + if self.raw_runtime_signature is not None: + namespace[FunctionContractAttribute.raw_runtime_signature] = self.raw_runtime_signature if self.input_memory_type is not None: MemoryContractAttribute.INPUT.write(namespace, self.input_memory_type) if self.output_memory_type is not None: @@ -826,10 +983,7 @@ def config_bound_parameters(self) -> tuple[inspect.Parameter, ...]: from openhcs.core.config import runtime_config_parameter - signature = inspect.signature( - self.resolve_canonical_raw_callable(), - eval_str=True, - ) + signature = self.canonical_signature return tuple( normalized for parameter in signature.parameters.values() @@ -892,7 +1046,7 @@ def declared_path_values( ) -> Mapping[str, tuple["PlatePathDeclaration", object]]: """Return authored or signature-default values for declared paths.""" - signature = inspect.signature(self.resolve_canonical_raw_callable()) + signature = self.canonical_signature values: dict[str, tuple["PlatePathDeclaration", object]] = {} for parameter_name, declaration in self.declared_path_parameters.items(): value = kwargs.get(parameter_name, signature.parameters[parameter_name].default) @@ -972,17 +1126,15 @@ def require_processing_contract(self) -> "ProcessingContract": ) return processing_contract - def raw_main_flow_call_argument(self, source_payload: Any) -> Any: + def runtime_main_flow_call_argument(self, source_payload: Any) -> Any: """Project this callable's ABI without discarding an adapter's context.""" - from arraybridge import ArrayPayload - from openhcs.core.runtime_image_values import image_payload_data - if self.runtime_adapter is not None: return source_payload - annotation = self.canonical_parameter_annotations.get(self.primary_input_parameter_name) - if isinstance(annotation, type) and issubclass(annotation, ArrayPayload): - return source_payload - return image_payload_data(source_payload) + return self.raw_main_flow_call_argument(source_payload) + + def raw_main_flow_call_argument(self, source_payload: Any) -> Any: + """Project the metadata-owned raw carrier declaration for this contract.""" + return self.metadata.raw_main_flow_call_argument(source_payload, self.canonical_signature) def main_flow_call_argument(self, source_payload: Any) -> Any: """Let the processing declaration retain context needed before raw calls.""" @@ -1050,14 +1202,8 @@ def artifact_specs(self) -> ArtifactSpecCollection: @property def primary_input_parameter_name(self) -> str | None: """FunctionStep input payload parameter declared by callable signature.""" - signature = inspect.signature(self.resolve_canonical_raw_callable()) - for parameter in signature.parameters.values(): - if parameter.kind in ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - ): - return parameter.name - return None + signature = self.canonical_signature + return self.metadata.primary_input_name(signature.parameters) @property def accepts_implicit_main_flow_input(self) -> bool: @@ -1095,12 +1241,8 @@ def resolve_runtime_callable(self) -> Callable[..., object]: return runtime_adapter.executable_callable(resolved, self) def resolve_canonical_raw_callable(self) -> Callable[..., object]: - """Return the declaration-owned callable whose signature defines behavior.""" - - declared = self.raw_processing_function - if declared is None: - declared = self.func - return _resolve_declared_callable(declared) + """Return the declaration-owned semantic target.""" + return self.metadata.resolve_canonical_raw_callable(self.func) def canonical_raw_import_identity(self) -> CallableImportIdentity: """Return the complete import identity of the canonical raw callable.""" @@ -1110,41 +1252,49 @@ def canonical_raw_import_identity(self) -> CallableImportIdentity: ) def resolve_raw_runtime_callable(self) -> Callable[..., object]: - """Remove runtime wrappers without crossing a semantic request binding.""" + """Resolve the metadata-owned raw target without crossing request binding.""" + return self.metadata.resolve_raw_runtime_callable(self.func) - canonical = self.resolve_canonical_raw_callable() - if self.request_binding is None: - return inspect.unwrap(canonical) + @property + def canonical_parameter_annotations(self) -> Mapping[str, object]: + """Derive annotations from the original semantic signature owner.""" + return { + name: parameter.annotation + for name, parameter in self.canonical_parameter_declarations.items() + } - binding_key = FunctionContractAttribute.callable_request_binding + @property + def canonical_parameter_declarations(self) -> Mapping[str, inspect.Parameter]: + """Resolve signature and annotations together at their declaration owner.""" + return self.canonical_signature.parameters - def is_request_boundary(candidate: Callable[..., object]) -> bool: - namespace = _callable_namespace(candidate) - if binding_key not in namespace: - return False - wrapped = namespace.get("__wrapped__") - return not callable(wrapped) or binding_key not in _callable_namespace( - wrapped - ) + @property + def canonical_signature(self) -> inspect.Signature: + """The metadata-owned semantic signature for this contract's target.""" + return self.metadata.canonical_signature_for(self.func) - resolved = inspect.unwrap(canonical, stop=is_request_boundary) - if binding_key not in _callable_namespace(resolved): - raise RuntimeError( - f"Callable {self.function_name!r} declares a request binding, but " - "its canonical raw callable has no request-binding boundary." - ) - return resolved + def with_prepared_signature(self) -> "CallableContract": + """Freeze the prepared declaration into its existing compiled contract.""" + return dataclasses.replace(self, metadata=self.metadata.with_prepared_signatures( + self.resolve_canonical_raw_callable(), self.resolve_raw_runtime_callable(), + )) @property - def canonical_parameter_annotations(self) -> Mapping[str, object]: - """Derive annotations from the original semantic signature owner.""" + def raw_runtime_signature(self) -> inspect.Signature: + """The metadata-owned execution signature for this contract's target.""" + return self.metadata.raw_runtime_signature_for(self.func) - raw_callable = self.resolve_canonical_raw_callable() - annotations = get_type_hints(raw_callable, include_extras=True) - return { - name: annotations.get(name, parameter.annotation) - for name, parameter in inspect.signature(raw_callable).parameters.items() - } + @classmethod + def from_prepared_callable(cls, func: Callable[..., object]) -> "CallableContract": + """Construct a compiled view only after the declaration owner prepares it.""" + projection = CallableProjection.from_callable(func) + prepared = cls.from_callable(projection.prepared_callable()) + return dataclasses.replace( + prepared, func=func, function_name=projection.name, module_name=projection.module_name, + metadata=prepared.metadata.with_missing_signatures( + prepared.resolve_canonical_raw_callable, prepared.resolve_raw_runtime_callable, + ), + ) def decode_public_kwargs(self, kwargs: Mapping[str, object]) -> dict[str, object]: """Descend declared scalar enums at authoring, before source projection.""" @@ -1172,7 +1322,7 @@ def validate_public_kwargs( raise TypeError( "CallableContract.validate_public_kwargs requires a mapping." ) - signature = inspect.signature(self.resolve_canonical_raw_callable()) + signature = self.canonical_signature runtime_owned_value = object() call_kwargs = dict(kwargs) runtime_loaded_parameters = frozenset(runtime_loaded_artifact_parameter_names) @@ -1243,15 +1393,19 @@ def validate_artifact_input_parameter_bindings(self) -> None: """Validate exact artifact occurrences against the normalized callable ABI.""" from openhcs.core.pipeline.function_contracts import ( - resolved_callable_parameter, special_input_parameter_accepts_sequence, ) - raw_callable = self.resolve_canonical_raw_callable() - artifact_parameters = { - parameter_name: resolved_callable_parameter(raw_callable, parameter_name) - for parameter_name in self.artifact_input_parameter_names - } + artifact_parameters = {} + parameters = self.canonical_parameter_declarations + for parameter_name in self.artifact_input_parameter_names: + parameter = parameters.get(parameter_name) + if parameter is None: + raise ValueError( + f"Callable {self.function_name!r} does not declare parameter " + f"{parameter_name!r}." + ) + artifact_parameters[parameter_name] = parameter specs_by_parameter: dict[str, list[ArtifactSpec]] = {} for spec in self.artifact_inputs: parameter_name = spec.parameter_name @@ -1468,6 +1622,8 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return func(**binding.implementation_kwargs(bound.arguments)) wrapper_namespace = _mutable_callable_namespace(wrapper) + wrapper_namespace.pop(FunctionContractAttribute.canonical_signature, None) + wrapper_namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) wrapper_namespace[FunctionContractAttribute.callable_request_binding] = binding wrapper_namespace["__signature__"] = public_signature return wrapper @@ -1522,6 +1678,8 @@ def attach_callable_contract_metadata( primary_image_carrier_transition: PrimaryImageCarrierTransition | None = None, ) -> None: """Attach OpenHCS callable metadata used by compiler/runtime phases.""" + _mutable_callable_namespace(func).pop(FunctionContractAttribute.canonical_signature, None) + _mutable_callable_namespace(func).pop(FunctionContractAttribute.raw_runtime_signature, None) if declared_processing_contract is not None: if ( not isinstance(declared_processing_contract, str) @@ -1698,9 +1856,10 @@ def attach_processing_prepare(func: Any, prepare: Any) -> None: f"got {type(prepare).__name__}." ) for target in CallableProjection.from_callable(func).prepare_targets(): - _mutable_callable_namespace(target)[ - FunctionContractAttribute.processing_prepare - ] = prepare + namespace = _mutable_callable_namespace(target) + namespace.pop(FunctionContractAttribute.canonical_signature, None) + namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) + namespace[FunctionContractAttribute.processing_prepare] = prepare def runtime_image_execution_mode( @@ -1852,6 +2011,21 @@ def from_callable(cls, func: Any) -> "CallableProjection": ) return cls(func=func, name=name, module_name=module_name, namespace=namespace) + def prepared_callable(self) -> Any: + """Keep ready library references or prepare live authored targets before validation.""" + reader = CallableMetadataReader(self.namespace, self.name) + if reader.has_prepared_signature(): + return self.func + resolved = _resolve_declared_callable(self.func) + for target in type(self).from_callable(resolved).prepare_targets(): + prepare_processing_callable(target) + return resolved + + def warm_canonical_signature(self) -> None: + """Publish prepared target signatures only after all library hooks finish.""" + contract = CallableContract.from_callable(self.func).with_prepared_signature() + contract.metadata.publish_signatures(_mutable_callable_namespace(self.func)) + def prepare_targets(self) -> tuple[Any, ...]: """Return wrapper/raw callables that may carry preparation metadata.""" targets: list[Any] = [] @@ -1877,6 +2051,7 @@ def prepare_targets(self) -> tuple[Any, ...]: def _runtime_context_parameter( projection: CallableProjection, reader: "CallableMetadataReader", + signature: inspect.Signature | None, ) -> str | None: declared = reader.optional_string( FunctionContractAttribute.runtime_context_parameter @@ -1885,19 +2060,8 @@ def _runtime_context_parameter( return declared from openhcs.core.context.processing_context import ProcessingContext - return _callable_signature_parameter( - projection.func, - ProcessingContext.require_parameter_name(), - ) - - -def _callable_signature_parameter(func: Any, parameter_name: str) -> str | None: - """Return a callable parameter name accepted by live signatures only.""" - if not callable(func): - return None - if parameter_name not in inspect.signature(func).parameters: - return None - return parameter_name + parameter_name = ProcessingContext.require_parameter_name() + return parameter_name if signature is not None and parameter_name in signature.parameters else None @dataclass(frozen=True, slots=True) @@ -1907,6 +2071,16 @@ class CallableMetadataReader: namespace: CallableNamespace function_name: str + def has_prepared_signature(self) -> bool: + """Validate the namespace-owned readiness declaration before phase selection.""" + return self.optional_signature(FunctionContractAttribute.canonical_signature) is not None + + def optional_signature(self, field_name: str) -> inspect.Signature | None: + value = self.namespace.get(field_name) + if value is not None and not isinstance(value, inspect.Signature): + raise TypeError(f"{self.function_name!r}.{field_name} must be an inspect.Signature or None.") + return value + def optional_string(self, field_name: str) -> str | None: """Return an optional string metadata field.""" value = self.namespace.get(field_name) @@ -2257,12 +2431,12 @@ def _artifact_input_parameter_names( def _artifact_input_specs_from_projection( projection: CallableProjection, artifact_inputs: tuple[ArtifactSpec, ...], + signature: inspect.Signature | None, ) -> tuple[ArtifactSpec, ...]: """Bind exact same-name input declarations at the metadata boundary.""" - if not callable(projection.func): + if signature is None: return artifact_inputs - signature = inspect.signature(projection.func) primary_parameter_name = next( ( parameter.name @@ -2292,6 +2466,7 @@ def _artifact_input_parameter_names_from_projection( projection: CallableProjection, reader: CallableMetadataReader, artifact_inputs: tuple[ArtifactSpec, ...], + signature: inspect.Signature | None, ) -> tuple[str, ...]: """Normalize compatibility and exact artifact parameter declarations once.""" @@ -2345,9 +2520,8 @@ def _artifact_input_parameter_names_from_projection( f"disagree: {declarations}." ) - if not callable(projection.func): + if signature is None: return first_names - signature = inspect.signature(projection.func) ordered_names = tuple(name for name in signature.parameters if name in first_set) return ( *ordered_names, diff --git a/openhcs/core/component_group_scope.py b/openhcs/core/component_group_scope.py index 78b18d51af..9a5ca314f5 100644 --- a/openhcs/core/component_group_scope.py +++ b/openhcs/core/component_group_scope.py @@ -10,6 +10,7 @@ from openhcs.constants.constants import AllComponents from openhcs.core.component_set import ComponentSet from openhcs.core.source_metadata import ( + SourceMetadataFields, SourceMetadataMapping, SourceMetadataScalar, SourceMetadataValue, @@ -467,7 +468,7 @@ def for_group_coordinate( def fixed_component_metadata( self, metadata: SourceMetadataMapping | None = None, - ) -> dict[str, SourceMetadataValue]: + ) -> SourceMetadataMapping: """Merge fixed execution coordinates into source component metadata.""" from openhcs.constants.constants import get_multiprocessing_axis @@ -476,7 +477,9 @@ def fixed_component_metadata( with_source_component_metadata, ) - merged: dict[str, SourceMetadataValue] = dict(metadata or {}) + merged = SourceMetadataFields.composition_snapshot( + metadata if metadata is not None else {} + ) fixed_values = ( (get_multiprocessing_axis(), self.axis_id), *self.fixed_component_values, diff --git a/openhcs/core/equivalence/relationships.py b/openhcs/core/equivalence/relationships.py index 1f03d975c8..4dbff95e2b 100644 --- a/openhcs/core/equivalence/relationships.py +++ b/openhcs/core/equivalence/relationships.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass, field -from typing import ClassVar +from typing import ClassVar, Self from metaclass_registry import RegistryFamily, RegistryKeyAttribute import numpy as np @@ -54,7 +54,7 @@ RuntimeMeasurementDialect, normalize_runtime_identifier, ) -from openhcs.core.equivalence.tables import measurement_table_padding_group +from openhcs.core.equivalence.tables import RuntimeTableSnapshot, measurement_table_padding_group from openhcs.core.registry_strategies import MostDerivedContextStrategyMixin from openhcs.core.measurement_row_materialization import ( columnar_row_values, @@ -66,9 +66,15 @@ MeasurementScope, MeasurementStatistic, ObjectCoreMeasurementFeature, + MeasurementScalarLiteral, ) from openhcs.core.runtime_relationships import ( ObjectInstanceKey, + ObjectRelationshipDeclaration, + DirectedObjectRelationshipPayload, + DirectParentReferenceFeatureDeclaration, + ChildCountFeatureDeclaration, + DirectParentReferenceMeasurementFeature, ObjectInstanceRelationship, ) from openhcs.core.runtime_tabular_values import ( @@ -1047,3 +1053,444 @@ def mean_child_value( if not values: return float("nan") return float(sum(values) / len(values)) + + +@dataclass(frozen=True, slots=True) +class ExportedRelationshipMeasurementSemantics(RelationshipMeasurementSemantics): + """Restore directed edge meaning using the saved object-row domains.""" + + object_tables: tuple[RuntimeScopedMeasurementTable, ...] + image_number_offset: RuntimeImageNumberOffset + + @classmethod + def supports_table(cls, table: RuntimeTableSnapshot) -> bool: + fields = frozenset(normalize_runtime_identifier(name) for name in table.header) + declaration_fields = frozenset( + ObjectRelationshipDeclaration.exported_field_names + ) + if not fields & declaration_fields: + return False + if not declaration_fields <= fields: + raise ValueError( + f"Relationship table {table.path} has an incomplete declaration schema." + ) + return True + + @classmethod + def validated_output_tables( + cls, + tables: tuple[RuntimeTableSnapshot, ...], + policy: RuntimeEquivalencePolicy, + ) -> tuple[RuntimeTableSnapshot, ...]: + """Consume every edge table before exposing its redundant scalar projection.""" + ordinary = tuple(table for table in tables if not cls.supports_table(table)) + object_tables = tuple( + RuntimeScopedMeasurementTable(measurement) + for table in ordinary + for measurement in table.measurement_tables(policy.measurement_dialect) + ) + image_offset = RuntimeImageNumberOffset.from_runtime_rows( + row + for table in object_tables + for row in table.table.rows.iter_row_mappings() + ) + relationships = tuple( + relationship + for table in tables + if cls.supports_table(table) + for relationship in cls.from_table( + table, object_tables, policy, image_offset + ) + ) + for relationship in relationships: + relationship.validate_recorded_measurements() + relationship.validate_reciprocal_declarations(relationships) + return ordinary + + @classmethod + def from_table( + cls, + table: RuntimeTableSnapshot, + object_tables: tuple[RuntimeScopedMeasurementTable, ...], + policy: RuntimeEquivalencePolicy, + image_number_offset: RuntimeImageNumberOffset, + ) -> tuple[Self, ...]: + normalized_header = tuple( + normalize_runtime_identifier(name) for name in table.header + ) + if len(set(normalized_header)) != len(normalized_header): + raise ValueError("Relationship table has duplicate normalized columns.") + groups: dict[ + ObjectRelationshipDeclaration, list[RuntimeMeasurementRowMapping] + ] = {} + for values in table.required_rows(): + row = RuntimeMeasurementRowMapping( + dict(zip(table.header, values, strict=True)) + ) + declaration = ObjectRelationshipDeclaration.from_exported_values( + *( + row.first_value((field,)) + for field in ObjectRelationshipDeclaration.exported_field_names + ) + ) + groups.setdefault(declaration, []).append(row) + results = [] + for declaration, rows in groups.items(): + source_ids = [] + target_ids = [] + slice_indices = [] + counts = set() + for row in rows: + source_id = MeasurementScalarLiteral( + row.first_value((declaration.source_id_field,)) + ).integer_value + target_id = MeasurementScalarLiteral( + row.first_value((declaration.target_id_field,)) + ).integer_value + if ( + source_id is None + or target_id is None + or min(source_id, target_id) < 0 + ): + raise ValueError( + "Relationship endpoint IDs must be nonnegative integers." + ) + slice_index = image_number_offset.object_instance_key( + row.row, 1 + ).required_slice_index() + raw_count = row.first_value(("slice_count",)) + count = MeasurementScalarLiteral(raw_count).integer_value + if raw_count is not None and count is None: + raise ValueError( + "Relationship slice count must be an integer when declared." + ) + counts.add(count) + source_ids.append(source_id) + target_ids.append(target_id) + slice_indices.append(slice_index) + if len(counts) != 1: + raise ValueError( + "Relationship rows have conflicting declared slice counts." + ) + relationship = ObjectRelationship( + name=declaration.artifact_name(), + declaration=declaration, + payload=DirectedObjectRelationshipPayload( + source_ids=tuple(source_ids), + target_ids=tuple(target_ids), + slice_indices=tuple(slice_indices), + slice_count=counts.pop(), + ), + ) + results.append( + cls( + relationship, + policy.measurement_dialect, + object_tables, + image_number_offset, + ) + ) + return tuple(results) + + @property + def source_name(self) -> str: + declaration = self.relationship.declaration + ref = ( + declaration.source + if declaration.projects_parent_child_measurements() + else declaration.target + ) + return normalize_runtime_identifier(ref.name) + + @property + def target_name(self) -> str: + declaration = self.relationship.declaration + ref = ( + declaration.target + if declaration.projects_parent_child_measurements() + else declaration.source + ) + return normalize_runtime_identifier(ref.name) + + @property + def instance_relationship(self) -> ObjectInstanceRelationship: + instances = super( + ExportedRelationshipMeasurementSemantics, self + ).instance_relationship + return ( + instances + if self.relationship.declaration.projects_parent_child_measurements() + else instances.reverse_endpoints() + ) + + def validate_reciprocal_declarations(self, relationships: tuple[Self, ...]) -> None: + """Require an observed reverse declaration to retain producer and pairs.""" + declaration = self.relationship.declaration + reverse = tuple( + other + for other in relationships + if other.source_subject == self.source_subject + and other.target_subject == self.target_subject + and other.relationship.declaration.source_role != declaration.source_role + ) + if reverse and not any( + other.relationship.declaration.producer_module_number + == declaration.producer_module_number + and other.instance_relationship.slice_count + == self.instance_relationship.slice_count + and frozenset( + zip( + other.instance_relationship.source_keys, + other.instance_relationship.target_keys, + strict=True, + ) + ) + == frozenset( + zip( + self.instance_relationship.source_keys, + self.instance_relationship.target_keys, + strict=True, + ) + ) + for other in reverse + ): + raise ValueError( + "Relationship reverse declarations disagree on producer or directed pairs." + ) + + @classmethod + def object_rows( + cls, + object_tables: tuple[RuntimeScopedMeasurementTable, ...], + subject: RuntimeMeasurementSubjectKey, + image_number_offset: RuntimeImageNumberOffset, + ) -> dict[ObjectInstanceKey, RuntimeMeasurementRowMapping]: + rows: dict[ObjectInstanceKey, RuntimeMeasurementRowMapping] = {} + for scoped_table in object_tables: + table = scoped_table.table + if ( + RuntimeMeasurementSubjectKey.from_table_subject(table.subject) + != subject + ): + continue + for values in table.rows.iter_row_mappings(): + row = RuntimeMeasurementRowMapping(measurement_row_mapping(values)) + object_id = row.object_label() + if object_id is None: + raise ValueError( + f"Relationship endpoint {subject.name!r} lacks an object identity." + ) + key = image_number_offset.object_instance_key(row.row, object_id) + key.required_slice_index() + if key in rows and rows[key].row != row.row: + raise ValueError( + f"Relationship endpoint {subject.name!r} has conflicting object rows for {key!r}." + ) + rows[key] = row + return rows + + def source_domain( + self, object_label_catalog: RuntimeObjectLabelInstanceCatalog + ) -> tuple[ObjectInstanceKey, ...]: + del object_label_catalog + return tuple( + self.object_rows( + self.object_tables, self.source_subject, self.image_number_offset + ) + ) + + def target_domain( + self, object_label_catalog: RuntimeObjectLabelInstanceCatalog + ) -> tuple[ObjectInstanceKey, ...]: + del object_label_catalog + return tuple( + self.object_rows( + self.object_tables, self.target_subject, self.image_number_offset + ) + ) + + def child_keys_by_parent( + self, object_label_catalog: RuntimeObjectLabelInstanceCatalog + ) -> dict[ObjectInstanceKey, tuple[ObjectInstanceKey, ...]]: + return self.instance_relationship.child_keys_by_parent( + declared_source_keys=self.source_domain(object_label_catalog) + ) + + def validate_recorded_measurements(self) -> None: + """Check all edges and zero-child rows before suppressing redundant facts.""" + parents = self.object_rows( + self.object_tables, self.source_subject, self.image_number_offset + ) + children = self.object_rows( + self.object_tables, self.target_subject, self.image_number_offset + ) + original = self.relationship.payload + for source_id, target_id, slice_index in zip( + original.source_ids, + original.target_ids, + original.slice_indices, + strict=True, + ): + parent_id, child_id = ( + (source_id, target_id) + if self.relationship.declaration.projects_parent_child_measurements() + else (target_id, source_id) + ) + if ( + child_id <= 0 + or ObjectInstanceKey(child_id, slice_index) not in children + ): + raise ValueError( + "Relationship table refers to an absent child endpoint." + ) + if ( + parent_id > 0 + and ObjectInstanceKey(parent_id, slice_index) not in parents + ): + raise ValueError( + "Relationship table refers to an absent parent endpoint." + ) + instances = self.instance_relationship + pairs = tuple(zip(instances.source_keys, instances.target_keys, strict=True)) + if len(set(pairs)) != len(pairs): + raise ValueError("Relationship table contains duplicate directed edges.") + parent_by_child = instances.parent_key_by_child() + if len(parent_by_child) != len(pairs): + raise ValueError( + "Relationship table assigns contradictory parents to one child." + ) + if ( + not set(instances.source_keys) <= parents.keys() + or not set(instances.target_keys) <= children.keys() + ): + raise ValueError("Relationship table refers to an absent object endpoint.") + parent_feature = DirectParentReferenceFeatureDeclaration.feature_name( + DirectParentReferenceMeasurementFeature( + self.relationship.declaration.source.name + if self.relationship.declaration.projects_parent_child_measurements() + else self.relationship.declaration.target.name + ) + ) + for child, row in children.items(): + observed = row.first_value((normalize_runtime_identifier(parent_feature),)) + if observed is None: + raise ValueError( + f"Relationship child {child!r} lacks its explicit parent measurement." + ) + expected = ( + parent_by_child[child].object_id if child in parent_by_child else 0 + ) + if MeasurementScalarLiteral(observed).integer_value != expected: + raise ValueError( + f"Relationship edges disagree with {parent_feature!r} at child {child!r}." + ) + counts = instances.child_keys_by_parent(declared_source_keys=parents) + count_feature = ChildCountFeatureDeclaration.feature_name( + self.relationship.declaration.target.name + if self.relationship.declaration.projects_parent_child_measurements() + else self.relationship.declaration.source.name + ) + for parent, row in parents.items(): + observed = row.first_value((normalize_runtime_identifier(count_feature),)) + if observed is None or MeasurementScalarLiteral( + observed + ).integer_value != len(counts[parent]): + raise ValueError( + f"Relationship edges disagree with {count_feature!r} at parent {parent!r}." + ) + + @classmethod + def correlated_object_relationships( + cls, + object_tables: tuple[RuntimeScopedMeasurementTable, ...], + image_number_offset: RuntimeImageNumberOffset, + ) -> dict[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship]: + """Retain exact native parent references, independently of value counters.""" + pairs: dict[ + RuntimeMeasurementFeatureKey, + list[tuple[ObjectInstanceKey, ObjectInstanceKey]], + ] = {} + for scoped_table in object_tables: + table = scoped_table.table + if table.subject.scope is not MeasurementScope.OBJECT: + continue + child_name = normalize_runtime_identifier(table.subject.name) + for field in table.rows.fields: + identity = DirectParentReferenceFeatureDeclaration.from_feature_name( + field.name + ) + if identity is None: + continue + key = RuntimeMeasurementFeatureKey( + RuntimeMeasurementSubjectKey( + MeasurementScope.RELATIONSHIP, child_name + ), + normalize_runtime_identifier(identity.parent_object_name), + ) + values = pairs.setdefault(key, []) + for raw_row in table.rows.iter_row_mappings(): + row = RuntimeMeasurementRowMapping(measurement_row_mapping(raw_row)) + child_id = row.object_label() + parent_id = MeasurementScalarLiteral( + row.row[field.name] + ).integer_value + if child_id is None or parent_id is None or parent_id < 0: + raise ValueError( + "Relationship measurements require integral endpoint identities." + ) + child_key = image_number_offset.object_instance_key( + row.row, child_id + ) + slice_index = child_key.required_slice_index() + if parent_id > 0: + values.append( + ( + ObjectInstanceKey(parent_id, slice_index), + child_key, + ) + ) + result = {} + for key, values in pairs.items(): + ordered = tuple( + sorted( + values, + key=lambda pair: ( + pair[0].slice_index, + pair[0].object_id, + pair[1].object_id, + ), + ) + ) + graph = ObjectInstanceRelationship( + tuple(pair[0] for pair in ordered), tuple(pair[1] for pair in ordered) + ) + if len(set(graph.target_keys)) != len(graph.target_keys): + raise ValueError( + "Relationship measurements assign duplicate or contradictory child identities." + ) + parent_subject = RuntimeMeasurementSubjectKey( + MeasurementScope.OBJECT, key.feature_name + ) + parents = cls.object_rows( + object_tables, parent_subject, image_number_offset + ) + if not set(graph.source_keys) <= parents.keys(): + raise ValueError( + "Relationship measurements refer to an absent parent endpoint." + ) + children_by_parent = graph.child_keys_by_parent( + declared_source_keys=parents + ) + count_field = normalize_runtime_identifier( + ChildCountFeatureDeclaration.feature_name(key.subject.name) + ) + for parent, row in parents.items(): + observed = row.first_value((count_field,)) + if observed is not None and MeasurementScalarLiteral( + observed + ).integer_value != len(children_by_parent[parent]): + raise ValueError( + f"Relationship measurements disagree with {count_field!r} at parent {parent!r}." + ) + result[key] = graph + return result diff --git a/openhcs/core/equivalence/tables.py b/openhcs/core/equivalence/tables.py index 05962edc89..4f947d719e 100644 --- a/openhcs/core/equivalence/tables.py +++ b/openhcs/core/equivalence/tables.py @@ -78,6 +78,12 @@ class RuntimeTableSnapshot: rows: tuple[tuple[str, ...], ...] column_context: tuple[str | None, ...] = () + def required_rows(self) -> tuple[tuple[str, ...], ...]: + """Admit actual rows when a consumer needs a determining declaration.""" + if not self.rows: + raise ValueError(f"Runtime table {self.path} has no determining declaration rows.") + return self.rows + @property def participates_in_comparison(self) -> bool: """Compare output tables except the engine's Experiment key/value receipt.""" diff --git a/openhcs/core/function_contract_metadata.py b/openhcs/core/function_contract_metadata.py index b269577cb8..09713a46ce 100644 --- a/openhcs/core/function_contract_metadata.py +++ b/openhcs/core/function_contract_metadata.py @@ -16,6 +16,8 @@ class FunctionContractAttribute: "__openhcs_declared_processing_contract__" ) raw_processing_function: ClassVar[str] = "__openhcs_raw_processing_function__" + canonical_signature: ClassVar[str] = "__openhcs_canonical_signature__" + raw_runtime_signature: ClassVar[str] = "__openhcs_raw_runtime_signature__" processing_prepare: ClassVar[str] = "__openhcs_prepare__" runtime_adapter: ClassVar[str] = "__runtime_adapter__" runtime_image_execution_mode: ClassVar[str] = ( diff --git a/openhcs/core/function_patterns.py b/openhcs/core/function_patterns.py index beb0b05af7..ea9a8abe82 100644 --- a/openhcs/core/function_patterns.py +++ b/openhcs/core/function_patterns.py @@ -1313,7 +1313,7 @@ def normalize( func, kwargs = _split_function_item(item) if RUNTIME_CALLABLE_KWARG_POLICY.item_is_disabled(kwargs): continue - contract = CallableContract.from_callable(func) + contract = CallableContract.from_prepared_callable(func) position = len(normalized_items) normalized_items.append( NormalizedFunctionItem( diff --git a/openhcs/core/orchestrator/compiled_plate_execution.py b/openhcs/core/orchestrator/compiled_plate_execution.py index f2e49a305c..a5fe8a7921 100644 --- a/openhcs/core/orchestrator/compiled_plate_execution.py +++ b/openhcs/core/orchestrator/compiled_plate_execution.py @@ -104,6 +104,9 @@ class CompiledPlateExecutionExtras: RESULTS_SUMMARY_KEY: ClassVar[str] = "viewer_states_by_port" viewer_states_by_port: Mapping[int, "ViewerControlResponse"] + runtime_observation: RuntimeExecutionObservation = field( + default_factory=RuntimeExecutionObservation + ) class CompiledPlateExecutionResults(dict[str, ExecutionResult]): @@ -120,6 +123,13 @@ def __init__( viewer_states_by_port=MappingProxyType({}) ) + @property + def runtime_observations(self) -> tuple[RuntimeExecutionObservation, ...]: + return ( + *tuple(result.runtime_observation for result in self.values()), + self.extras.runtime_observation, + ) + @dataclass(frozen=True, slots=True) class CompiledPlateExecutionRequest(ProgressExecutionContext): @@ -249,6 +259,7 @@ def execute_compiled_plate_request( executor_resources.clear_execution_bundle() executor_resources.release_parent_runtime_resources(execution_bundle) + plate_runtime_observation = RuntimeExecutionObservation() if all(result.is_success() for result in execution_results.values()): plate_runtime_observation = execute_plate_scoped_steps( validated.compiled_contexts, @@ -285,7 +296,8 @@ def execute_compiled_plate_request( return CompiledPlateExecutionResults( execution_results, extras=CompiledPlateExecutionExtras( - viewer_states_by_port=viewer_states_by_port + viewer_states_by_port=viewer_states_by_port, + runtime_observation=plate_runtime_observation, ), ) except ExecutionCancelledError: @@ -445,6 +457,7 @@ def execute_plate_scoped_steps( plate_step_indexes = validate_plate_scoped_contexts(compiled_contexts) total_steps = max(next(iter(compiled_contexts.values())).step_plans) + 1 records_by_axis = _runtime_record_snapshot(compiled_contexts) + runtime_export_paths_by_context = {key: [] for key in compiled_contexts} for step_index in plate_step_indexes: owner_context, owner_plan = _plate_output_owner( compiled_contexts, @@ -523,13 +536,24 @@ def execute_plate_scoped_steps( ) records_by_axis = _records_with_output(records_by_axis, record) - RuntimeArtifactMaterializationAuthority.materialize( + materializations = RuntimeArtifactMaterializationAuthority.materialize( owner_context, owner_plan.require_function_execution_ready(), ) OpenHCSMetadataWriter.write( owner_context, owner_plan.require_function_execution_ready(), + artifact_materializations=materializations, + ) + owner_context_key = next( + key + for key, context in compiled_contexts.items() + if context is owner_context + ) + runtime_export_paths_by_context[owner_context_key].extend( + path + for materialization in materializations + for path in materialization.observation(owner_plan).runtime_export_paths ) _emit_execution_progress( progress_queue=progress_queue, @@ -547,6 +571,9 @@ def execute_plate_scoped_steps( RuntimeContextObservation( context_key=context_key, records=records, + runtime_export_paths=tuple( + dict.fromkeys(runtime_export_paths_by_context[context_key]) + ), ) for context_key, context in compiled_contexts.items() if ( diff --git a/openhcs/core/orchestrator/worker_execution.py b/openhcs/core/orchestrator/worker_execution.py index 7f82c7bafc..fe7776e554 100644 --- a/openhcs/core/orchestrator/worker_execution.py +++ b/openhcs/core/orchestrator/worker_execution.py @@ -48,11 +48,15 @@ from openhcs.core.progress.runtime_artifacts import ( runtime_artifact_context_for_records, ) -from openhcs.core.runtime_stores import StoredRuntimeValue +from openhcs.core.runtime_stores import ( + StoredRuntimeValue, + RuntimeArtifactAddress, + RuntimeArtifactLocation, +) from openhcs.core.steps.abstract import AbstractStep, StepExecutionObservation from openhcs.core.steps.function_artifact_materialization import ( - observed_materialized_artifact_locations_by_address, - observed_runtime_export_artifact_output_paths, + preview_reused_materialized_artifact_locations, + preview_reused_runtime_export_paths, ) from openhcs.utils.environment import OpenHCSProcessEnvironment @@ -63,21 +67,16 @@ def _runtime_observation_progress_context( records: tuple[StoredRuntimeValue, ...], *, - plan: CompiledStepPlan, - context: ProcessingContext, + materialized_locations_by_address: Mapping[ + RuntimeArtifactAddress, tuple[RuntimeArtifactLocation, ...] + ], ) -> dict | None: """Project one RuntimeValueStore observation delta through owned payloads.""" runtime_artifacts = runtime_artifact_context_for_records(records) live_measurements = live_measurement_context_for_records( records, - materialized_locations_by_address=( - observed_materialized_artifact_locations_by_address( - plan, - context, - records, - ) - ), + materialized_locations_by_address=materialized_locations_by_address, ) if runtime_artifacts is None: return live_measurements @@ -795,22 +794,16 @@ def _execute_axis_with_sequential_combinations( pipeline_definition, frozen_context, lane_context, + context_key=context_key, cancellation=cancellation, ) observed_records = runtime_store.observed_values_after( execution_observation_cursor ) runtime_export_paths = tuple( - dict.fromkeys( - path - for step_plan in frozen_context.step_plans.values() - if step_plan.owns_runtime_outputs - for path in observed_runtime_export_artifact_output_paths( - step_plan, - frozen_context, - observed_records, - ) - ) + path + for observation in result.runtime_observation.contexts + for path in observation.runtime_export_paths ) finally: # This cache is context-local even when lanes share a process. @@ -932,6 +925,8 @@ def _execute_single_axis_static( pipeline_definition: List[AbstractStep], frozen_context: ProcessingContext, lane_context: WorkerLaneExecutionContext, + *, + context_key: str, cancellation: ExecutionCancellationSignal | None = None, ) -> ExecutionResult: """Execute one frozen axis context against the compiled pipeline.""" @@ -952,6 +947,7 @@ def _execute_single_axis_static( frozen_context.bind_execution_runtime(lane_context) lane_context.install_debug_sink(frozen_context) runtime_value_store = frozen_context.runtime_value_store + runtime_export_paths = [] for step_index, step in enumerate(pipeline_definition): if cancellation is not None: @@ -981,8 +977,15 @@ def _execute_single_axis_static( ) runtime_progress_context = _runtime_observation_progress_context( observed_records, - plan=step_plan, - context=frozen_context, + # Historical debug previews are not a new-save observation. + materialized_locations_by_address=preview_reused_materialized_artifact_locations( + step_plan, frozen_context, observed_records + ), + ) + runtime_export_paths.extend( + preview_reused_runtime_export_paths( + step_plan, frozen_context, observed_records + ) ) emit( execution_id=lane_context.execution_id, @@ -1016,12 +1019,12 @@ def _execute_single_axis_static( ) observation_cursor = runtime_value_store.observation_cursor() - step.process(frozen_context, step_index) + step_observation = step.process(frozen_context, step_index) + runtime_export_paths.extend(step_observation.runtime_export_paths) observed_records = runtime_value_store.observed_values_after(observation_cursor) runtime_progress_context = _runtime_observation_progress_context( observed_records, - plan=step_plan, - context=frozen_context, + materialized_locations_by_address=step_observation.materialized_locations_by_address, ) emit( @@ -1044,7 +1047,18 @@ def _execute_single_axis_static( ): break - return ExecutionResult.success(axis_id=axis_id) + return ExecutionResult.success( + axis_id=axis_id, + runtime_observation=RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + context_key=context_key, + records=(), + runtime_export_paths=tuple(dict.fromkeys(runtime_export_paths)), + ), + ) + ), + ) def execute_worker_lane( diff --git a/openhcs/core/pipeline/function_contracts.py b/openhcs/core/pipeline/function_contracts.py index f966e0e8d2..e0d343fc5f 100644 --- a/openhcs/core/pipeline/function_contracts.py +++ b/openhcs/core/pipeline/function_contracts.py @@ -3,7 +3,6 @@ from collections.abc import Mapping, Sequence from dataclasses import replace from enum import Enum -from functools import lru_cache import inspect from types import UnionType from typing import ( @@ -45,11 +44,19 @@ F = TypeVar("F", bound=Callable) -@lru_cache(maxsize=256) def resolved_callable_type_hints(func: Callable) -> dict[str, Any]: """Return the callable's resolved type contract or propagate its error.""" - return get_type_hints(func) + signature = CallableMetadata.prepared_callable_signature(func) + if signature is None: + return get_type_hints(func) + from typing import _strip_annotations + + return { + **{name: _strip_annotations(parameter.annotation) for name, parameter in signature.parameters.items() + if parameter.annotation is not inspect.Parameter.empty}, + **({} if signature.return_annotation is inspect.Signature.empty else {"return": _strip_annotations(signature.return_annotation)}), + } def annotation_accepts_runtime_type(annotation: object, value_type: type[Any]) -> bool: @@ -103,7 +110,7 @@ def resolved_callable_parameter( ) -> inspect.Parameter: """Return one callable parameter with its runtime type hint resolved.""" - signature = inspect.signature(func) + signature = CallableMetadata.callable_signature(func) parameter = signature.parameters.get(parameter_name) if parameter is None: raise ValueError( @@ -131,7 +138,6 @@ def preserves_full_stack(self, *, image_stack_required: bool) -> bool: self is self.MATCH_IMAGE_STACK and image_stack_required ) - def invocation_kwargs( self, kwargs: Mapping[str, Any], @@ -460,7 +466,7 @@ def special_input_parameters_from_callable( """Return special-input parameters in their canonical signature order.""" declared_names = special_input_names_from_callable(func) - signature = inspect.signature(func) + signature = CallableMetadata.callable_signature(func) missing = tuple(name for name in declared_names if name not in signature.parameters) if missing: raise ValueError( diff --git a/openhcs/core/roi_point_metadata.py b/openhcs/core/roi_point_metadata.py index f597dd2bec..90ef89c57e 100644 --- a/openhcs/core/roi_point_metadata.py +++ b/openhcs/core/roi_point_metadata.py @@ -11,6 +11,8 @@ from openhcs.constants.constants import AllComponents from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.source_image_provenance import SourceComponentMetadata +from openhcs.core.source_matching import source_component_metadata_items +from openhcs.core.source_metadata import SourceVoxelSpacing @dataclass(frozen=True, slots=True) @@ -83,11 +85,24 @@ def source_component_domain( raise ValueError( "3D point ROI Z coordinate lies outside its source planes." ) - common_keys = set(domain[0]) - {z_field} - if any(set(plane) - {z_field} != common_keys for plane in domain): + components = tuple( + dict(source_component_metadata_items(plane)) for plane in domain + ) + common_keys = set(components[0]) - {AllComponents.Z_INDEX} + if any( + set(plane) - {AllComponents.Z_INDEX} != common_keys + for plane in components + ): raise ValueError("3D point ROI source planes have inconsistent components.") if any( - any(plane[key] != domain[0][key] for key in common_keys) for plane in domain + any(plane[key] != components[0][key] for key in common_keys) + for plane in components ): raise ValueError("3D point ROI source planes vary outside Z.") + spacing = SourceVoxelSpacing.from_source_metadata(domain[0]) + if any( + SourceVoxelSpacing.from_source_metadata(plane) != spacing + for plane in domain + ): + raise ValueError("3D point ROI source planes have inconsistent calibration.") return tuple(domain) diff --git a/openhcs/core/roi_source_metadata.py b/openhcs/core/roi_source_metadata.py index 6455aaadf5..f6d90351fc 100644 --- a/openhcs/core/roi_source_metadata.py +++ b/openhcs/core/roi_source_metadata.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import replace from typing import ClassVar @@ -40,13 +40,11 @@ def decode(cls, rois: Sequence[ROI]) -> ImagePayloadMetadata | None: def geometry(cls, rois: Sequence[ROI]) -> list[ROI]: """Keep transport-only source declarations out of the ROI feature table.""" return [ - replace( - roi, - metadata={ - key: value - for key, value in roi.metadata.items() - if key != cls.FIELD - }, - ) + replace(roi, metadata=cls.feature_metadata(roi.metadata)) for roi in rois ] + + @classmethod + def feature_metadata(cls, metadata: Mapping[str, object]) -> dict[str, object]: + """Project features without discarding the transported source record.""" + return {key: value for key, value in metadata.items() if key != cls.FIELD} diff --git a/openhcs/core/runtime_adapters.py b/openhcs/core/runtime_adapters.py index 14e0bb43cc..e8b2d3cf2a 100644 --- a/openhcs/core/runtime_adapters.py +++ b/openhcs/core/runtime_adapters.py @@ -42,7 +42,6 @@ ) from openhcs.core.source_binding_selection import ( SourceBindingMatchedImageSet, - SourcePatternResolutionContext, ) from openhcs.core.source_image_provenance import ( SourceImageIdentity, @@ -305,10 +304,12 @@ def source_artifact_payload(self, ref: ArtifactSpecRef) -> object: projection, axis_id=self.axis_scope.axis_id, ) - source_context = SourcePatternResolutionContext.from_projection( - parser=self.context.microscope_handler.parser, - projection=projection, - metadata_rules=self.source_binding_plan.metadata_rules, + source_context = ( + self.context.runtime_source_binding_context_cache.source_pattern_context( + parser=self.context.microscope_handler.parser, + projection=projection, + metadata_rules=self.source_binding_plan.metadata_rules, + ) ) matched_set = SourceBindingMatchedImageSet.from_plan( bindings=self.source_binding_plan.binding_declarations, @@ -475,7 +476,9 @@ class RuntimeAdapterSpec: parameter_name: str factory: RuntimeAdapterFactory manages_artifact_inputs: bool = False - artifact_output_policy: type[ArtifactOutputPolicy] = NativeReturnArtifactOutputPolicy + artifact_output_policy: type[ArtifactOutputPolicy] = ( + NativeReturnArtifactOutputPolicy + ) runtime_callable_factory: RuntimeCallableFactory | None = None def __post_init__(self) -> None: @@ -530,7 +533,9 @@ def runtime_adapter( factory: RuntimeAdapterFactory, *, manages_artifact_inputs: bool = False, - artifact_output_policy: type[ArtifactOutputPolicy] = NativeReturnArtifactOutputPolicy, + artifact_output_policy: type[ + ArtifactOutputPolicy + ] = NativeReturnArtifactOutputPolicy, runtime_callable_factory: RuntimeCallableFactory | None = None, ) -> Callable[[_F], _F]: """Declare that a callable needs an invocation-scoped runtime adapter.""" diff --git a/openhcs/core/runtime_batch_contracts.py b/openhcs/core/runtime_batch_contracts.py index 6933013d51..d3eef11fcc 100644 --- a/openhcs/core/runtime_batch_contracts.py +++ b/openhcs/core/runtime_batch_contracts.py @@ -6,14 +6,13 @@ from collections.abc import Callable, Hashable, Mapping from dataclasses import dataclass, replace from enum import Enum -from functools import lru_cache import inspect from types import MappingProxyType from typing import ClassVar, Generic, TypeVar from metaclass_registry import AutoRegisterMeta -from openhcs.core.callable_contract import KeywordRuntimeParameter +from openhcs.core.callable_contract import CallableMetadata, KeywordRuntimeParameter from openhcs.core.aligned_image_payload import ( AlignedImageStack, ImagePayloadExecutionMode, @@ -48,11 +47,12 @@ class RuntimePlaneAxisValueProjectionParameter(KeywordRuntimeParameter): parameter_default = None -@lru_cache(maxsize=1024) -def runtime_callable_defaults(func: Callable[..., object]) -> Mapping[str, object]: +def runtime_callable_defaults( + func: Callable[..., object], *, signature: inspect.Signature | None = None, +) -> Mapping[str, object]: """Return callable defaults visible to runtime batch executors.""" try: - callable_signature = inspect.signature(func) + callable_signature = CallableMetadata.callable_signature(func) if signature is None else signature except (TypeError, ValueError) as exc: raise TypeError( f"Runtime batch function {func!r} must expose an inspectable signature." @@ -190,6 +190,7 @@ class RuntimePure2DSliceBatchRequest( ], RuntimeSliceResultT, ] + signature: inspect.Signature | None = None def __post_init__(self) -> None: object.__setattr__( @@ -197,7 +198,7 @@ def __post_init__(self) -> None: "kwargs", MappingProxyType( { - **runtime_callable_defaults(self.func), + **runtime_callable_defaults(self.func, signature=self.signature), **dict(self.kwargs), } ), @@ -279,6 +280,14 @@ def executors(self) -> Mapping[RuntimeBatchExecutionDomain, Callable]: class Pure2DSliceBatchExecutor(RuntimeBatchExecutor): """Base contract for equivalent pure-2D slice batch execution.""" + @classmethod + def from_executors( + cls, executors: Mapping[RuntimeBatchExecutionDomain, Callable] | None, + ) -> Callable: + """Honor a declared pure-2D executor before the nominal serial default.""" + declared = None if executors is None else executors.get(RuntimeBatchExecutionDomain.PURE_2D_SLICES) + return declared if callable(declared) else cls.default_executor() + @classmethod def default_executor(cls) -> "Pure2DSliceBatchExecutor": """Return the single-thread default pure-2D batch executor.""" diff --git a/openhcs/core/runtime_equivalence.py b/openhcs/core/runtime_equivalence.py index 244f9edde5..4fbbbe3d2f 100644 --- a/openhcs/core/runtime_equivalence.py +++ b/openhcs/core/runtime_equivalence.py @@ -69,6 +69,8 @@ MeasurementTable, ) from openhcs.core.runtime_relationships import ( + ObjectInstanceKey, + ObjectInstanceRelationship, ObjectRelationship, ) from openhcs.core.equivalence.policy import ( @@ -141,6 +143,7 @@ from openhcs.core.equivalence.relationships import ( RelationshipAggregateFeatureSemantics, RelationshipMeasurementSemantics, + ExportedRelationshipMeasurementSemantics, RuntimeObjectRelationshipIdentity, RuntimeScopedMeasurementTable, object_measurement_values_by_label, @@ -1708,7 +1711,7 @@ def project_measurement_fact_counts(self) -> RuntimeMeasurementFactCounterMap: str | None, ] RuntimeCellSignatureCachePayload = tuple[str, str] -RuntimeMeasurementSnapshotCachePayload = tuple[ +RuntimeMeasurementFactCachePayload = tuple[ tuple[ RuntimeMeasurementFeatureCachePayload, tuple[tuple[RuntimeCellSignatureCachePayload, int], ...], @@ -1717,11 +1720,24 @@ def project_measurement_fact_counts(self) -> RuntimeMeasurementFactCounterMap: ] +RuntimeRelationshipCorrelationCachePayload = tuple[ + tuple[RuntimeMeasurementFeatureCachePayload, tuple[tuple[tuple[int, int | None], tuple[int, int | None]], ...], int | None], + ..., +] +RuntimeMeasurementSnapshotCachePayload = tuple[ + RuntimeMeasurementFactCachePayload, + RuntimeRelationshipCorrelationCachePayload | None, +] + + @dataclass(slots=True) class RuntimeMeasurementSnapshot: """Semantic measurement facts independent of table layout.""" measurement_fact_counts: RuntimeMeasurementFactCounterMapping + correlated_relationships: ( + Mapping[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship] | None + ) = None @classmethod def from_output_snapshot( @@ -1736,18 +1752,29 @@ def from_output_snapshot( policy=policy, known_source_names=known_source_names, ) - state.record_measurement_tables( - tuple( - RuntimeScopedMeasurementTable(measurement_table) - for table in snapshot.tables - for measurement_table in table.measurement_tables( - policy.measurement_dialect - ) - ), - None, + tables = tuple( + RuntimeScopedMeasurementTable(measurement_table) + for table in ExportedRelationshipMeasurementSemantics.validated_output_tables( + snapshot.tables, policy + ) + for measurement_table in table.measurement_tables( + policy.measurement_dialect + ) ) + image_offset = RuntimeImageNumberOffset.from_runtime_rows( + row for table in tables for row in table.table.rows.iter_row_mappings() + ) + correlations = ( + ExportedRelationshipMeasurementSemantics.correlated_object_relationships( + tables, image_offset + ) + ) + state.record_measurement_tables(tables, None) state.required_measurement_keys = frozenset(state.explicit_measurement_keys) - return cls(measurement_fact_counts=state.project_measurement_fact_counts()) + return cls( + measurement_fact_counts=state.project_measurement_fact_counts(), + correlated_relationships=correlations, + ) @classmethod def from_artifact_execution_observation( @@ -1779,48 +1806,110 @@ def __post_init__(self) -> None: } ) + if self.correlated_relationships is not None: + self.correlated_relationships = MappingProxyType( + dict(self.correlated_relationships) + ) + @property def is_empty(self) -> bool: return not self.measurement_fact_counts - def to_cache_payload( + def required_relationship_correlations( self, - ) -> RuntimeMeasurementSnapshotCachePayload: - """Return a stable semantic cache payload for repeated equivalence checks.""" - return tuple( + ) -> Mapping[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship]: + """Admit this snapshot for a full saved comparison, not value-only scope.""" + if self.correlated_relationships is None: + raise RuntimeError( + "Matched saved output inventory requires known relationship correlations." + ) + return self.correlated_relationships + + def relationship_differences( + self, + candidate: ( + Mapping[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship] | None + ), + ) -> tuple[RuntimeEquivalenceDifference, ...]: + """Compare saved correlations; None denotes the typed value-only scope.""" + if self.correlated_relationships is None or candidate is None: + return () + if self.correlated_relationships == candidate: + return () + return ( + RuntimeEquivalenceDifference( + RuntimeEquivalenceDifferenceKind.MEASUREMENT_CONTENT, + "directed relationship endpoint/image correlations differ", + ), + ) + + def to_cache_payload(self) -> RuntimeMeasurementSnapshotCachePayload: + """Transport numeric facts and known/unknown relationship evidence together.""" + facts = tuple( ( feature.to_cache_payload(), tuple( (value.to_cache_payload(), int(count)) for value, count in sorted( - values.items(), - key=lambda item: item[0].sort_key, + values.items(), key=lambda item: item[0].sort_key ) ), ) for feature, values in sorted( - self.measurement_fact_counts.items(), - key=lambda item: item[0].sort_key, + self.measurement_fact_counts.items(), key=lambda item: item[0].sort_key ) ) + relationships = None + if self.correlated_relationships is not None: + relationships = tuple( + ( + key.to_cache_payload(), + tuple( + ( + (source.object_id, source.slice_index), + (target.object_id, target.slice_index), + ) + for source, target in zip( + value.source_keys, value.target_keys, strict=True + ) + ), + value.slice_count, + ) + for key, value in sorted( + self.correlated_relationships.items(), + key=lambda item: item[0].sort_key, + ) + ) + return facts, relationships @classmethod def from_cache_payload( - cls, - payload: RuntimeMeasurementSnapshotCachePayload, + cls, payload: RuntimeMeasurementSnapshotCachePayload ) -> "RuntimeMeasurementSnapshot": - """Rebuild a semantic measurement snapshot from cache payload data.""" - measurement_fact_counts: RuntimeMeasurementFactCounterMap = {} - for feature_payload, values_payload in payload: # type: ignore[union-attr] - counter: Counter[RuntimeCellSignature] = Counter() - for value_payload, count in values_payload: - counter[RuntimeCellSignature.from_cache_payload(value_payload)] = int( - count + """Restore the same evidence scope, retaining every directed endpoint pair.""" + fact_payload, relationship_payload = payload + facts = { + RuntimeMeasurementFeatureKey.from_cache_payload(feature): Counter( + { + RuntimeCellSignature.from_cache_payload(value): count + for value, count in values + } + ) + for feature, values in fact_payload + } + relationships = None + if relationship_payload is not None: + relationships = { + RuntimeMeasurementFeatureKey.from_cache_payload( + key + ): ObjectInstanceRelationship( + tuple(ObjectInstanceKey(*source) for source, _target in pairs), + tuple(ObjectInstanceKey(*target) for _source, target in pairs), + slice_count, ) - measurement_fact_counts[ - RuntimeMeasurementFeatureKey.from_cache_payload(feature_payload) - ] = counter - return cls(measurement_fact_counts=measurement_fact_counts) + for key, pairs, slice_count in relationship_payload + } + return cls(facts, relationships) @dataclass(slots=True) @@ -1881,9 +1970,18 @@ def runtime_measurement_equivalence( *, policy: RuntimeEquivalencePolicy = RuntimeEquivalencePolicy(), ) -> RuntimeEquivalenceReport: - """Compare precomputed semantic measurement snapshots.""" + """Compare measurement facts and any jointly supplied saved edge correlations. + + Typed artifact snapshots retain their value-only scope when global saved + image numbering is unavailable. Full saved qualification must use the + known correlation evidence produced by ``from_output_snapshot`` on both + sides; an UNKNOWN typed projection is not such a qualification. + """ return RuntimeEquivalenceReport( - differences=_measurement_differences(reference, candidate, policy) + differences=( + *_measurement_differences(reference, candidate, policy), + *reference.relationship_differences(candidate.correlated_relationships), + ) ) diff --git a/openhcs/core/runtime_execution_validation.py b/openhcs/core/runtime_execution_validation.py index 8f1c90d4a1..b47296cc06 100644 --- a/openhcs/core/runtime_execution_validation.py +++ b/openhcs/core/runtime_execution_validation.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections import Counter -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from types import MappingProxyType @@ -12,6 +12,7 @@ from openhcs.core.artifacts import ArtifactSpec, ArtifactType from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.orchestrator.execution_result import RuntimeExecutionObservation from openhcs.core.runtime_exports import ( RuntimeExportExpectation, RuntimeExportObservation, @@ -235,12 +236,13 @@ def runtime_artifact_viewer_expectations( if not plan.streaming_configs or not plan.owns_runtime_outputs: continue from openhcs.core.steps.function_artifact_materialization import ( - observed_runtime_artifact_materializations, + runtime_artifact_materializations_from_records, ) - for materialization in observed_runtime_artifact_materializations( + for materialization in runtime_artifact_materializations_from_records( plan, context, + context.runtime_value_store.observed_values, ): output_plan = materialization.output_plan if plan.compiled_function_pattern.publishes_output_to_main_flow( @@ -319,6 +321,8 @@ class RuntimeArtifactExecutionObservation: def from_contexts( cls, execution_contexts: Mapping[str, ProcessingContext], + *, + runtime_observations: Sequence[RuntimeExecutionObservation], ) -> "RuntimeArtifactExecutionObservation": identity_policies = frozenset( context.source_image_set_identity_policy @@ -331,8 +335,8 @@ def from_contexts( ) return cls( records_by_axis=runtime_records_by_axis(execution_contexts), - exports=RuntimeExportObservation.from_execution_contexts( - execution_contexts + exports=RuntimeExportObservation.from_runtime_observations( + runtime_observations ), source_image_set_identity_policy=next( iter(identity_policies), diff --git a/openhcs/core/runtime_exports.py b/openhcs/core/runtime_exports.py index 1cc9d50317..ee5f64cc05 100644 --- a/openhcs/core/runtime_exports.py +++ b/openhcs/core/runtime_exports.py @@ -28,7 +28,6 @@ from openhcs.processing.materialization.core import materialization_is_empty if TYPE_CHECKING: - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.orchestrator.execution_result import RuntimeExecutionObservation @@ -113,26 +112,6 @@ class RuntimeExportObservation: table_row_counts_by_path: Mapping[Path, int] output_files: tuple[Path, ...] = () - @classmethod - def from_execution_contexts( - cls, - execution_contexts: Mapping[str, ProcessingContext], - ) -> RuntimeExportObservation: - """Read contract-owned export paths without retaining runtime values.""" - from openhcs.core.steps.function_artifact_materialization import ( - runtime_export_artifact_output_paths, - ) - - return cls.from_output_paths( - tuple( - path - for context in execution_contexts.values() - for plan in context.step_plans.values() - if plan.owns_runtime_outputs - for path in runtime_export_artifact_output_paths(plan, context) - ) - ) - @classmethod def from_runtime_observations( cls, diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index f43374ff3d..661367cdef 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -47,6 +47,8 @@ ) from openhcs.core.source_metadata import ( SourceMetadataScalar, + OwnedSourceMetadataFields, + SourceMetadataFields, SourceMetadataValue, SourceVoxelSpacing, SourceVoxelSpacingFields, @@ -251,15 +253,28 @@ def for_array_payload( ) def __post_init__(self, *source_provenance_values: object) -> None: - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) + self.normalize_metadata_fields() + + def normalize_metadata_fields(self) -> None: + """Normalize this metadata's typed fields in their constructor effect order.""" self.source_voxel_spacing = self.source_voxel_spacing.with_missing_from( SourceVoxelSpacing.from_source_metadata(self.source_component_metadata) ) self.normalize_source_provenance_fields() self.normalize_source_spatial_domain_fields() self.normalize_source_voxel_spacing_fields() + self.validate_source_channel_axis() + if self.plane_axis is not None: + self.plane_axis = RuntimePlaneAxis(self.plane_axis) + + def require_leading_plane_axis(self, message: str) -> None: + """Require axis presence before later ordered projection validation.""" + if self.plane_axis is None: + raise ValueError(message) + + def validate_source_channel_axis(self) -> None: + """Validate the authored channel declaration before transforming axes.""" if self.source_channel_axis is not None and ( not isinstance(self.source_channel_axis, int) or isinstance(self.source_channel_axis, bool) @@ -267,10 +282,6 @@ def __post_init__(self, *source_provenance_values: object) -> None: raise TypeError( "ImagePayloadMetadata.source_channel_axis must be int or None." ) - if self.plane_axis is not None: - self.plane_axis = RuntimePlaneAxis( - self.plane_axis, - ) @property def has_values(self) -> bool: @@ -600,16 +611,15 @@ def without_source_channel_axis(self) -> "ImagePayloadMetadata": def for_leading_source_plane(self, plane_index: int) -> "ImagePayloadMetadata": """Project metadata after explicitly removing its leading plane axis.""" - if self.plane_axis is None: - raise ValueError( - "Leading source-plane projection requires a declared plane axis." - ) - return self.for_source_plane(plane_index).without_leading_plane_axis() + return LeadingSourcePlaneMetadataProjection(self, plane_index).project() - def without_leading_plane_axis(self) -> "ImagePayloadMetadata": - """Return metadata after an explicitly declared leading axis is removed.""" - if self.plane_axis is None: - raise ValueError("Image metadata has no leading plane axis to remove.") + def without_leading_plane_axis( + self, *, projection: "ImageMetadataProjection | None" = None + ) -> "ImagePayloadMetadata": + """Remove an axis through the projection's declared result ownership.""" + self.require_leading_plane_axis( + "Image metadata has no leading plane axis to remove." + ) source_channel_axis = self.source_channel_axis if source_channel_axis == 0: raise ValueError( @@ -618,19 +628,18 @@ def without_leading_plane_axis(self) -> "ImagePayloadMetadata": ) if source_channel_axis is not None and source_channel_axis > 0: source_channel_axis -= 1 - return self.with_source_provenance( - self.source_provenance.with_runtime_planes_as_contributors() - ).replace_fields( - plane_axis=None, - source_channel_axis=source_channel_axis, - source_plane_intensity_scales=(), - source_plane_dtypes=(), - unit_interval_intensity=( - None - if self.unit_interval_intensity is None - else self.unit_interval_intensity.without_source_planes() - ), - ) + if projection is None: + projection = LeadingPlaneAxisMetadataProjection(self) + projected = projection.project_source_provenance( + self, self.source_provenance.with_runtime_planes_as_contributors() + ) + projected.plane_axis = None + projected.source_channel_axis = source_channel_axis + projected.source_plane_intensity_scales = () + projected.source_plane_dtypes = () + projected.unit_interval_intensity = self.project_intensity_proof(None) + projected.normalize_metadata_fields() + return projected def collapse_leading_plane_axis(self) -> "ImagePayloadMetadata": """Return scalar metadata after reducing every plane of the leading axis.""" @@ -666,22 +675,27 @@ def source_image_paths(self) -> tuple[str, ...]: def intensity_scale_for_source_plane(self, plane_index: int) -> float | None: """Return the best available intensity scale for one source plane.""" - if 0 <= plane_index < len(self.source_plane_intensity_scales): - plane_scale = self.source_plane_intensity_scales[plane_index] - if plane_scale is not None: - return plane_scale - return self.intensity_scale + plane_scale = _tuple_value(self.source_plane_intensity_scales, plane_index) + return self.intensity_scale if plane_scale is None else plane_scale def unit_interval_intensity_scale_for_source_plane( self, plane_index: int, ) -> int | None: """Return the scale proving current pixels are exact integer/scale values.""" - if 0 <= plane_index < len(self.source_plane_unit_interval_intensity_scales): - plane_scale = self.source_plane_unit_interval_intensity_scales[plane_index] - if plane_scale is not None: - return int(plane_scale) - return self.unit_interval_intensity_scale + if self.unit_interval_intensity is None: + return None + return self.unit_interval_intensity.scale_for_source_plane(plane_index) + + def project_intensity_proof( + self, plane_index: int | None + ) -> ImageUnitIntervalIntensityMetadata | None: + """Select a plane's proof, or retain only the proof for the whole image.""" + if self.unit_interval_intensity is None: + return None + if plane_index is None: + return self.unit_interval_intensity.without_source_planes() + return self.unit_interval_intensity.for_source_plane(plane_index) @property def source_plane_metadata_count(self) -> int: @@ -737,20 +751,7 @@ def payload_with(self, data: Any, mask: Any | None = None) -> Any: def for_source_plane(self, plane_index: int) -> "ImagePayloadMetadata": """Return metadata for one source plane sliced from a stacked payload.""" - source_provenance = self.source_provenance.for_source_plane(plane_index) - return self.replace_fields( - intensity_scale=self.intensity_scale_for_source_plane(plane_index), - source_dtype=_tuple_value(self.source_plane_dtypes, plane_index) - or self.source_dtype, - source_provenance=source_provenance, - unit_interval_intensity=( - None - if self.unit_interval_intensity is None - else self.unit_interval_intensity.for_source_plane(plane_index) - ), - source_plane_intensity_scales=(), - source_plane_dtypes=(), - ) + return SourcePlaneImageMetadataProjection(self, plane_index).project() def for_source_planes( self, @@ -1398,6 +1399,96 @@ def with_image_payload_data( return resolved_metadata.payload_with(data, resolved_mask) +@dataclass(frozen=True, slots=True) +class ImageMetadataProjection(ABC): + """Construct one metadata result from nominal source and axis transformations.""" + + metadata: ImagePayloadMetadata + + def project(self) -> ImagePayloadMetadata: + return self.project_axis_metadata(self.project_source_metadata()) + + @abstractmethod + def project_source_metadata(self) -> ImagePayloadMetadata: + """Select source metadata with this projection's declared ownership.""" + + def project_axis_metadata( + self, metadata: ImagePayloadMetadata + ) -> ImagePayloadMetadata: + """Retain declared axes unless a nominal axis capability transforms them.""" + return metadata + + @abstractmethod + def project_source_provenance( + self, metadata: ImagePayloadMetadata, provenance: SourceImageProvenance + ) -> ImagePayloadMetadata: + """Apply provenance using the result ownership established by source selection.""" + + +@dataclass(frozen=True, slots=True) +class SourcePlaneImageMetadataProjection(ImageMetadataProjection): + """Select scalar source-image facts without removing an array axis.""" + + plane_index: int + + def project_source_metadata(self) -> ImagePayloadMetadata: + provenance = self.metadata.source_provenance.for_source_plane(self.plane_index) + return self.metadata.replace_fields( + intensity_scale=self.metadata.intensity_scale_for_source_plane( + self.plane_index + ), + source_dtype=_tuple_value( + self.metadata.source_plane_dtypes, self.plane_index + ) + or self.metadata.source_dtype, + source_provenance=provenance, + unit_interval_intensity=self.metadata.project_intensity_proof( + self.plane_index + ), + source_plane_intensity_scales=(), + source_plane_dtypes=(), + ) + + def project_source_provenance( + self, metadata: ImagePayloadMetadata, provenance: SourceImageProvenance + ) -> ImagePayloadMetadata: + """Normalize the independently owned result created by source selection.""" + metadata.source_provenance = provenance + metadata.normalize_metadata_fields() + return metadata + + +class LeadingPlaneAxisMetadataProjection(ImageMetadataProjection): + """Remove a declared leading axis while preserving its source contributors.""" + + def project_source_metadata(self) -> ImagePayloadMetadata: + return self.metadata + + def project_axis_metadata( + self, metadata: ImagePayloadMetadata + ) -> ImagePayloadMetadata: + return metadata.without_leading_plane_axis(projection=self) + + def project_source_provenance( + self, metadata: ImagePayloadMetadata, provenance: SourceImageProvenance + ) -> ImagePayloadMetadata: + """Create the standalone result after its axis guards and source derivation.""" + return metadata.replace_fields(source_provenance=provenance) + + +class LeadingSourcePlaneMetadataProjection( + SourcePlaneImageMetadataProjection, + LeadingPlaneAxisMetadataProjection, +): + """Remove the leading axis of independently owned selected-source metadata.""" + + def project_source_metadata(self) -> ImagePayloadMetadata: + self.metadata.require_leading_plane_axis( + "Leading source-plane projection requires a declared plane axis." + ) + return super().project_source_metadata() + + @dataclass(frozen=True, slots=True) class ImagePayloadSliceProjector: """Project payload context from a parent image into one child image slice.""" @@ -1762,7 +1853,9 @@ def common_source_component_metadata( """Return source metadata shared by the composed payload.""" metadata_values = tuple(values) metadata_by_plane = tuple( - dict(metadata.source_component_metadata) + SourceMetadataFields.composition_snapshot( + metadata.source_component_metadata + ) for metadata in metadata_values if metadata.source_component_metadata is not None ) @@ -1793,7 +1886,21 @@ def common_source_component_metadata( ) if not common_metadata: return None - return MappingProxyType(common_metadata) + owner = ( + metadata_by_plane[0] + if metadata_by_plane + and all( + isinstance(metadata, OwnedSourceMetadataFields) + for metadata in metadata_by_plane + ) + else {} + ) + derived = SourceMetadataFields.derived_mapping(owner, common_metadata) + return ( + derived + if isinstance(derived, OwnedSourceMetadataFields) + else MappingProxyType(derived) + ) @staticmethod def common_metadata_value( diff --git a/openhcs/core/runtime_measurements.py b/openhcs/core/runtime_measurements.py index b0c613097d..a213bd62d8 100644 --- a/openhcs/core/runtime_measurements.py +++ b/openhcs/core/runtime_measurements.py @@ -15,7 +15,6 @@ MeasurementObjectRowIdentity, ) from openhcs.core.source_image_provenance import ( - SourceImageProvenance, SourceImageProvenanceFields, ) @@ -75,9 +74,7 @@ def iter_row_mappings(self) -> Iterable[Mapping[str, object]]: return self.rows.iter_row_mappings() def __post_init__(self, *source_provenance_values: object) -> None: - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_source_provenance_fields() self.validate_artifact_name() if self.source_image_name == "": diff --git a/openhcs/core/runtime_object_label_building.py b/openhcs/core/runtime_object_label_building.py index cac8c54e0a..e47b3b06a2 100644 --- a/openhcs/core/runtime_object_label_building.py +++ b/openhcs/core/runtime_object_label_building.py @@ -142,18 +142,9 @@ def plane_semantics(self) -> RuntimePlaneAxisValueProjection | None: ) return None metadata = self.metadata - declared_projection = RuntimePlaneAxisValueProjection.from_source_declaration( + projection.validate_source_declaration( metadata.plane_axis, metadata.source_provenance, ) - if declared_projection is not None: - if ( - projection.axis is not declared_projection.axis - or projection.axis_size != declared_projection.axis_size - ): - raise ValueError( - "Object-label plane projection conflicts with the source-image " - "axis declaration." - ) label_array = np.asarray(self.labels) projection.validate_shape( label_array.shape, diff --git a/openhcs/core/runtime_object_labels.py b/openhcs/core/runtime_object_labels.py index 69e07dbd46..396e677ae6 100644 --- a/openhcs/core/runtime_object_labels.py +++ b/openhcs/core/runtime_object_labels.py @@ -6,6 +6,7 @@ from collections.abc import ( Callable, Hashable, + Iterable, MutableMapping, Sequence, ) @@ -14,6 +15,7 @@ import numpy as np from metaclass_registry import AutoRegisterMeta +from numba import njit from openhcs.core import ( runtime_array_values, @@ -28,6 +30,7 @@ NominalTypeStrategyFamilyMixin, ) from openhcs.core.runtime_object_label_domains import ( + DenseIntegerObjectLabelIdDomain, ObjectLabelDomain, ObjectLabelDomainDeclaration, ObjectLabelDomainMetadata, @@ -928,9 +931,7 @@ class ObjectLabelPayload(ObjectLabelValue): def __post_init__(self, *source_provenance_values: object) -> None: self.validate_object_label_variants() - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_object_label_metadata("ObjectLabelPayload") @@ -994,9 +995,7 @@ def from_payload( def __post_init__(self, *source_provenance_values: object) -> None: self.validate_object_label_variants() - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.validate_artifact_name() if self.source_image_name == "": raise ValueError("ObjectLabelSet.source_image_name cannot be empty.") @@ -1115,14 +1114,30 @@ def axis_centers( if sparse_labels.has_slice_index else (sparse_labels.y_column, sparse_labels.x_column) ) + return self.coordinate_centers( + counts, + ( + np.bincount( + object_ids, + weights=array[:, coordinate_column], + minlength=max_label + 1, + ) + for coordinate_column in coordinate_columns + ), + maximum_label=max_label, + ) + + @staticmethod + def coordinate_centers( + counts: np.ndarray, + coordinate_sums: Iterable[np.ndarray], + *, + maximum_label: int, + ) -> tuple[tuple[np.ndarray, ...], np.ndarray]: + """Allocate and divide each coordinate sum in its declared ID domain.""" axis_centers: list[np.ndarray] = [] - for coordinate_column in coordinate_columns: - sums = np.bincount( - object_ids, - weights=array[:, coordinate_column], - minlength=max_label + 1, - ) - centers = np.full(max_label + 1, np.nan, dtype=np.float64) + for sums in coordinate_sums: + centers = np.full(maximum_label + 1, np.nan, dtype=np.float64) np.divide(sums, counts, out=centers, where=counts > 0) axis_centers.append(centers) return tuple(axis_centers), counts @@ -1202,6 +1217,62 @@ def dense_data( def sparse_ijv_rows(self, labels: object) -> SparseIJVLabelRows: return SparseIJVLabelRows.from_dense_stack(cast(np.ndarray, labels)) + @classmethod + def prepare_coordinates(cls) -> None: + """Prepare C/F/strided signatures with both input mutability policies.""" + plane = np.array([[0, 1], [1, 0]], dtype=np.int32) + volume = np.stack((plane, plane)) + for labels in (volume, np.asfortranarray(volume), volume.copy()[..., ::-1]): + for writeable in (True, False): + labels.flags.writeable = writeable + _dense_label_coordinate_moments_numba(labels, 1) + for labels in (plane, np.asfortranarray(plane), plane.copy()[:, ::-1]): + for writeable in (True, False): + labels.flags.writeable = writeable + dense_label_centers_2d_numba(labels, 1) + + def axis_centers( + self, labels: object, *, domain: Sequence[int] + ) -> tuple[tuple[np.ndarray, ...], np.ndarray]: + array = np.asarray(labels) + if ( + array.dtype != np.dtype(np.int32) + or array.ndim not in (2, 3) + or any(size > np.iinfo(np.int32).max for size in array.shape) + ): + return super().axis_centers(labels, domain=domain) + coordinate_planes = array[None, ...] if array.ndim == 2 else array + positive_parts = tuple(plane[plane > 0] for plane in coordinate_planes) + coordinate_domain = DenseIntegerObjectLabelIdDomain.from_array( + np.concatenate(positive_parts or (np.empty(0, dtype=np.int32),)) + ) + del positive_parts + if coordinate_domain is None: + return super().axis_centers(labels, domain=domain) + coordinate_columns = tuple(range(3 - array.ndim, 3)) + sums, pixel_counts = _dense_label_coordinate_moments_numba( + coordinate_planes, coordinate_domain.max_label, + ) + del coordinate_domain + # Like sparse conversion, the moments snapshot precedes domain callbacks. + object_ids = np.flatnonzero(pixel_counts) + max_domain_label = max(domain, default=0) + maximum_label = max(int(object_ids.max(initial=0)), max_domain_label) + counts = np.bincount(object_ids, minlength=maximum_label + 1) + counts[object_ids] = pixel_counts[object_ids] + return self.coordinate_centers( + counts, + ( + np.bincount( + object_ids, + weights=sums[object_ids, coordinate_column], + minlength=maximum_label + 1, + ) + for coordinate_column in coordinate_columns + ), + maximum_label=maximum_label, + ) + def stack_planes( self, labels: Sequence[object], @@ -1403,6 +1474,14 @@ def sparse_ijv_rows(self, labels: object) -> SparseIJVLabelRows: label_data ) + def axis_centers( + self, labels: object, *, domain: Sequence[int] + ) -> tuple[tuple[np.ndarray, ...], np.ndarray]: + label_data = self.label_data(labels) + return ObjectLabelStorageStrategy.for_value(label_data).axis_centers( + label_data, domain=domain + ) + def stack_planes( self, labels: Sequence[object], @@ -1555,6 +1634,46 @@ def object_label_axis_centers( ) +@njit(cache=True) +def _dense_label_coordinate_moments_numba( + labels: np.ndarray, + maximum_label: int, +) -> tuple[np.ndarray, np.ndarray]: + """Reduce dense positive labels once in their declared row-major geometry.""" + sums = np.zeros((maximum_label + 1, 3), dtype=np.float64) + counts = np.zeros(maximum_label + 1, dtype=np.int64) + plane_count, height, width = labels.shape + for plane in range(plane_count): + for y in range(height): + for x in range(width): + label_id = int(labels[plane, y, x]) + if label_id > 0 and label_id <= maximum_label: + sums[label_id, 0] += plane + sums[label_id, 1] += y + sums[label_id, 2] += x + counts[label_id] += 1 + return sums, counts + + +@njit(cache=True) +def dense_label_centers_2d_numba( + labels: np.ndarray, label_count: int +) -> np.ndarray: + """Validate compiled two-dimensional geometry and return its y/x centers.""" + height, width = labels.shape + sums, counts = _dense_label_coordinate_moments_numba( + labels[None, :height, :width], label_count + ) + centers = np.empty((label_count + 1, 2), dtype=np.float64) + for label_id in range(label_count + 1): + for axis in range(2): + centers[label_id, axis] = ( + np.nan if counts[label_id] == 0 + else sums[label_id, axis + 1] / counts[label_id] + ) + return centers + + def object_label_storage_is_sparse_ijv(payload: object) -> bool: """Return whether the nominal storage authority owns sparse-IJV rows.""" authority = ObjectLabelStorageStrategy.for_value(payload) diff --git a/openhcs/core/runtime_output_matching.py b/openhcs/core/runtime_output_matching.py index 44cc127d39..7e6dd98b49 100644 --- a/openhcs/core/runtime_output_matching.py +++ b/openhcs/core/runtime_output_matching.py @@ -9,6 +9,7 @@ from openhcs.core.aligned_image_payload import ( AlignedImageSliceContext, AlignedImageStack, + pack_aligned_image_outputs, ) from openhcs.core.artifacts import ( ArtifactOutputPlan, @@ -16,6 +17,12 @@ ArtifactSpecRef, ) from openhcs.core.callable_contract import CallableContract +from openhcs.core.runtime_image_values import image_payload_metadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxisValueProjection +from openhcs.core.runtime_slice_projection import ( + RuntimeSliceProjection, + RuntimeSliceProjectionDeclarationError, +) class RuntimeOutputBundle(ABC): @@ -61,6 +68,75 @@ def canonical_output(self) -> Any: return split_runtime_output(self.returned_output)[0] + def contextualize_canonical_output( + self, + *, + plane_projection: RuntimePlaneAxisValueProjection | None = None, + ) -> Any: + """Attach the compiled canonical ABI to one exact returned image axis.""" + + canonical_specs = self.callable_contract.canonical_return_output_specs.specs + if len(canonical_specs) <= 1: + return self.returned_output + canonical_output, trailing_outputs = split_runtime_output(self.returned_output) + if isinstance(canonical_output, AlignedImageStack): + if canonical_output.slice_contexts: + return self.returned_output + output_values = canonical_output.slices + else: + projection = plane_projection + function_name = self.callable_contract.function_name + if projection is None: + raise RuntimeSliceProjectionDeclarationError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs but returned a non-aligned payload without a " + "compiled plane projection." + ) + if projection.plane_index is not None: + raise RuntimeSliceProjectionDeclarationError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs after the compiled plane projection already selected " + f"plane {projection.plane_index}." + ) + if projection.axis_size != len(canonical_specs): + raise ValueError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs but its compiled plane projection declares " + f"{projection.axis_size} value(s)." + ) + output_axis = image_payload_metadata(canonical_output).plane_axis + if output_axis is not projection.axis: + raise RuntimeSliceProjectionDeclarationError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs but its returned payload does not declare the " + f"compiled {projection.axis.value!r} plane axis; got " + f"{output_axis!r}." + ) + output_values = tuple( + RuntimeSliceProjection.value_for_slice( + canonical_output, + projection.selected_plane(output_index), + ) + for output_index in range(projection.axis_size) + ) + if len(output_values) != len(canonical_specs): + raise ValueError( + f"{self.callable_contract.function_name} returned {len(output_values)} " + "canonical output value(s) for " + f"{len(canonical_specs)} compiled output spec(s)." + ) + contextualized_output = pack_aligned_image_outputs( + output_values, + slice_contexts=AlignedImageSliceContext.main_flow_for_artifact_specs( + canonical_specs + ), + ) + return ( + (contextualized_output, *trailing_outputs) + if trailing_outputs + else contextualized_output + ) + def resolve(self) -> dict[ArtifactSpecRef, Any]: """Resolve every output in the callable ABI.""" @@ -85,12 +161,18 @@ def resolve(self) -> dict[ArtifactSpecRef, Any]: strict=True, ) } - resolved.update( - self._canonical_values( - self.callable_contract.canonical_return_output_specs.specs, - canonical_output, + canonical_specs = self.callable_contract.canonical_return_output_specs.specs + if len(canonical_specs) == 1: + resolved[canonical_specs[0].ref()] = canonical_output + elif canonical_specs: + if not isinstance(canonical_output, AlignedImageStack): + raise TypeError( + "Multiple canonical output specs require an AlignedImageStack with " + "one exact named slice context per output." + ) + resolved.update( + canonical_output.output_values_for_artifact_specs(canonical_specs) ) - ) return resolved def resolve_plan_values( @@ -129,70 +211,6 @@ def resolve_plan_values( matched_outputs.append((plan, spec, returned_values[ref])) return returned_values, tuple(matched_outputs) - @classmethod - def _canonical_values( - cls, - canonical_specs: tuple[ArtifactSpec, ...], - canonical_output: Any, - ) -> dict[ArtifactSpecRef, Any]: - """Resolve named outputs represented by the one canonical return slot.""" - - if not canonical_specs: - return {} - if len(canonical_specs) == 1: - spec = canonical_specs[0] - return {spec.ref(): canonical_output} - if not isinstance(canonical_output, AlignedImageStack): - raise TypeError( - "Multiple canonical output specs require an AlignedImageStack with " - "one exact named slice context per output." - ) - if not canonical_output.slice_contexts: - raise ValueError( - "Multiple canonical output specs require exact AlignedImageStack " - "slice contexts; positional slice order is not artifact identity." - ) - - specs_by_context = { - (spec.name, spec.artifact_type.value): spec for spec in canonical_specs - } - if len(specs_by_context) != len(canonical_specs): - raise ValueError("Canonical output ABI contains duplicate named contexts.") - resolved: dict[ArtifactSpecRef, Any] = {} - for payload, context in zip( - canonical_output.slices, - canonical_output.slice_contexts, - strict=True, - ): - if context.output_kind != AlignedImageSliceContext.MAIN_FLOW_OUTPUT_KIND: - raise ValueError( - "Canonical AlignedImageStack contains a non-main-flow slice " - f"context: {context!r}." - ) - context_key = (context.output_key, context.artifact_kind) - spec = specs_by_context.get(context_key) - if spec is None: - raise ValueError( - "Canonical AlignedImageStack context is not declared by the " - f"callable ABI: {context!r}." - ) - ref = spec.ref() - if ref in resolved: - raise ValueError( - "Canonical AlignedImageStack contains duplicate context for " - f"{ref!r}." - ) - resolved[ref] = payload - missing = tuple( - spec.ref() for spec in canonical_specs if spec.ref() not in resolved - ) - if missing: - raise ValueError( - "Canonical AlignedImageStack does not carry every declared output: " - f"{missing!r}." - ) - return resolved - @staticmethod def _specs_by_ref( specs: tuple[ArtifactSpec, ...], diff --git a/openhcs/core/runtime_plane_projection.py b/openhcs/core/runtime_plane_projection.py index fe1bf18bc0..9f7affad17 100644 --- a/openhcs/core/runtime_plane_projection.py +++ b/openhcs/core/runtime_plane_projection.py @@ -297,6 +297,21 @@ def from_projector( axis_size=axis_size, ) + def validate_source_declaration( + self, axis: RuntimePlaneAxis | None, source: "SourceImageProvenance", + ) -> None: + """Check an explicit projection against retained acquisition declarations.""" + if axis is None: + return + declared_count = source.source_plane_count + if self.axis is not axis or ( + declared_count > 0 and self.axis_size != declared_count + ): + raise ValueError( + "Object-label plane projection conflicts with the source-image " + "axis declaration." + ) + @classmethod def require_from_projector( cls, diff --git a/openhcs/core/runtime_relationships.py b/openhcs/core/runtime_relationships.py index 4abca79cfa..dfadd0035c 100644 --- a/openhcs/core/runtime_relationships.py +++ b/openhcs/core/runtime_relationships.py @@ -27,7 +27,12 @@ from abc import abstractmethod from collections.abc import Iterable from openhcs.core.registry_strategies import MostDerivedContextStrategyMixin -from openhcs.core.runtime_measurements import MeasurementRowAxisField +from openhcs.core.runtime_measurements import ( + MeasurementRowAxisField, + ObjectReferenceFeatureMarker, + RuntimeMeasurementFeatureDeclaration, + MeasurementScalarLiteral, +) from openhcs.core.source_spatial_domain import SourceSpatialDomainAdapter from openhcs.core.runtime_object_label_domains import ObjectLabelDomainMetadataStrategy from openhcs.core.runtime_object_label_domains import ObjectLabelDomainScope @@ -41,10 +46,106 @@ from openhcs.core.runtime_sparse_labels import SparseIJVLabelRows +class DirectParentReferenceFeatureMarker(ObjectReferenceFeatureMarker): + """Semantic marker for a child's direct parent-object reference.""" + + +@dataclass(frozen=True, slots=True) +class DirectParentReferenceMeasurementFeature: + """Nominal identity encoded by a ``Parent_`` measurement name.""" + + parent_object_name: str + + def __post_init__(self) -> None: + if not isinstance(self.parent_object_name, str) or not self.parent_object_name: + raise ValueError("Direct parent-reference object name cannot be empty.") + + +class DirectParentReferenceFeatureDeclaration(RuntimeMeasurementFeatureDeclaration): + """Parse and render direct parent references at their row-production owner.""" + + declaration_key = "direct_parent_reference" + semantic_marker_types = (DirectParentReferenceFeatureMarker,) + prefix = "Parent_" + + @classmethod + def from_feature_name( + cls, + feature_name: str, + ) -> DirectParentReferenceMeasurementFeature | None: + if not feature_name.startswith(cls.prefix): + return None + parent_object_name = feature_name[len(cls.prefix) :] + if not parent_object_name: + return None + return DirectParentReferenceMeasurementFeature(parent_object_name) + + @classmethod + def feature_name(cls, identity: object) -> str: + if not isinstance(identity, DirectParentReferenceMeasurementFeature): + raise TypeError( + f"{cls.__name__}.feature_name requires " + "DirectParentReferenceMeasurementFeature." + ) + return f"{cls.prefix}{identity.parent_object_name}" + + +class ChildCountFeatureDeclaration(RuntimeMeasurementFeatureDeclaration): + """Own the child-name grammar shared by export readers and CP producers.""" + + declaration_key = "child_count_reference" + prefix = "Children_" + suffix = "_Count" + + @classmethod + def from_feature_name(cls, feature_name: str) -> str | None: + if not feature_name.startswith(cls.prefix) or not feature_name.endswith( + cls.suffix + ): + return None + name = feature_name[len(cls.prefix) : -len(cls.suffix)].strip() + return name or None + + @classmethod + def feature_name(cls, identity: str) -> str: + name = identity.strip() + if not name: + raise ValueError( + "Child-count feature requires a non-empty child object name." + ) + return f"{cls.prefix}{name}{cls.suffix}" + + @dataclass(frozen=True) class ObjectRelationshipDeclaration(ArtifactSpecRelation): """Authoritative directed relationship semantics for one artifact output.""" + exported_field_names: ClassVar[tuple[str, ...]] = ( + "relationship_type", + "source_role", + "target_role", + "source_object", + "target_object", + "producer_module_number", + ) + + def exported_columns(self) -> dict[str, object]: + """Render the declaration owned by every exported edge row.""" + return dict( + zip( + self.exported_field_names, + ( + self.relationship_type, + self.source_role, + self.target_role, + self.source.name, + self.target.name, + self.producer_module_number, + ), + strict=True, + ) + ) + relation_key = "object_relationship_declaration" target_plan_type: ClassVar[type[ArtifactPlan] | None] = None target: ArtifactSpecRef @@ -122,6 +223,45 @@ def parent_child( target_runtime_slice_offset=target_runtime_slice_offset, ) + @classmethod + def from_exported_values( + cls, + relationship_type: str, + source_role: str, + target_role: str, + source_object: str, + target_object: str, + producer_module_number: object, + ) -> Self: + """Restore the complete declared parent/child edge grammar from CSV.""" + source = ArtifactSpec.output(source_object, ObjectLabelsArtifactType).ref() + target = ArtifactSpec.output(target_object, ObjectLabelsArtifactType).ref() + module = MeasurementScalarLiteral(producer_module_number).integer_value + if module is None: + raise ValueError("Relationship producer module number is required.") + canonical = cls.parent_child( + source=source, target=target, producer_module_number=module + ) + roles = (source_role, target_role) + canonical_roles = (canonical.source_role, canonical.target_role) + fields = (canonical.source_id_field, canonical.target_id_field) + if roles == canonical_roles: + source_field, target_field = fields + elif roles == canonical_roles[::-1]: + target_field, source_field = fields + else: + raise ValueError(f"Unsupported exported relationship roles {roles!r}.") + return cls( + source=source, + target=target, + relationship_type=relationship_type, + source_role=roles[0], + target_role=roles[1], + source_id_field=source_field, + target_id_field=target_field, + producer_module_number=module, + ) + def require_target_spec(self, spec: ArtifactSpec) -> None: super().require_target_spec(spec) if not issubclass(spec.artifact_type, ObjectLineageArtifactType): @@ -218,9 +358,7 @@ def from_payload( def __post_init__(self, *source_provenance_values: object) -> None: self.validate_artifact_name() - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_source_provenance_fields() if not isinstance(self.declaration, ObjectRelationshipDeclaration): raise TypeError( @@ -246,16 +384,9 @@ def project_runtime_slice(self, slice_index: int) -> "ObjectRelationship": def relationship_columns(self) -> dict[str, Any]: """Return semantic relationship columns without payload provenance.""" - columns = { - "relationship_type": self.declaration.relationship_type, - "source_role": self.declaration.source_role, - "target_role": self.declaration.target_role, - "source_object": self.declaration.source.name, - "target_object": self.declaration.target.name, - "producer_module_number": self.declaration.producer_module_number, - self.declaration.source_id_field: self.payload.source_ids, - self.declaration.target_id_field: self.payload.target_ids, - } + columns = self.declaration.exported_columns() + columns[self.declaration.source_id_field] = self.payload.source_ids + columns[self.declaration.target_id_field] = self.payload.target_ids if self.payload.slice_indices: columns["slice_index"] = self.payload.slice_indices if self.payload.slice_count is not None: @@ -820,6 +951,12 @@ def __post_init__(self) -> None: object.__setattr__(self, "object_id", object_id) object.__setattr__(self, "slice_index", slice_index) + def required_slice_index(self) -> int: + """Admit this object identity for a comparison requiring an explicit plane.""" + if self.slice_index is None: + raise ValueError("Object instance requires an explicit image or slice identity.") + return self.slice_index + @classmethod def from_measurement_row( cls, @@ -953,6 +1090,10 @@ def child_keys_by_parent( ) } + def reverse_endpoints(self) -> Self: + """Transpose the same directed instance pairs without losing their axes.""" + return type(self)(self.target_keys, self.source_keys, self.slice_count) + def parent_key_by_child(self) -> dict[ObjectInstanceKey, ObjectInstanceKey]: """Return source identity for each target identity.""" return dict(zip(self.target_keys, self.source_keys, strict=True)) diff --git a/openhcs/core/runtime_source_binding_cache.py b/openhcs/core/runtime_source_binding_cache.py index 7e2ffb242d..8b074a55c7 100644 --- a/openhcs/core/runtime_source_binding_cache.py +++ b/openhcs/core/runtime_source_binding_cache.py @@ -3,17 +3,87 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any, TYPE_CHECKING +from dataclasses import dataclass, field, replace +from types import MappingProxyType +from typing import Any, Hashable, TYPE_CHECKING from openhcs.core.source_bindings import ( SourceBindingRuntimeContext, SourceBindingRuntimeMetadataNormalizer, + MetadataExtractionRule, +) +from openhcs.core.source_metadata import ( + ResolvedSourceMetadataRecord, + SourceMetadataMapping, ) -from openhcs.core.source_metadata import SourceMetadataMapping if TYPE_CHECKING: - from openhcs.core.source_binding_selection import SourceUniverseRuntimeState + from openhcs.core.source_binding_selection import ( + SourcePatternResolutionContext, + SourceUniverseRuntimeState, + ) + from openhcs.core.source_workspace_projection import ( + VirtualWorkspaceSourceProjection, + ) + from openhcs.microscopes.microscope_interfaces import FilenameParser + + +@dataclass(frozen=True, slots=True) +class RuntimeSourceResolutionSnapshot: + """A derived resolution view retaining its exact projection and parser owners.""" + + projection: "VirtualWorkspaceSourceProjection" + context: "SourcePatternResolutionContext" + + @classmethod + def from_projection( + cls, + *, + parser: "FilenameParser", + projection: "VirtualWorkspaceSourceProjection", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> "RuntimeSourceResolutionSnapshot": + """Resolve declared positions into one independently owned runtime view.""" + from openhcs.core.source_binding_selection import ( + DeclaredSourceMetadataRecord, + SourcePatternResolutionContext, + ) + + context = SourcePatternResolutionContext.from_projection( + parser=parser, + projection=projection, + metadata_rules=metadata_rules, + ) + normalized_metadata = SourceBindingRuntimeMetadataNormalizer( + projection.source_metadata_by_path + ).normalized() + context = replace( + context, + source_metadata_by_path=MappingProxyType( + { + path: DeclaredSourceMetadataRecord.from_mapping(metadata) + for path, metadata in normalized_metadata.items() + } + ), + source_projections_by_virtual_path=MappingProxyType( + dict(context.source_projections_by_virtual_path) + ), + ) + paths = dict.fromkeys( + ( + *context.source_metadata_by_path, + *context.source_paths_by_virtual_path, + *context.source_paths_by_virtual_path.values(), + ) + ) + records = {} + for path in paths: + metadata = context.metadata_for_path(path) + records[path] = ResolvedSourceMetadataRecord.from_mapping( + {} if metadata is None else metadata + ) + context = replace(context, source_metadata_by_path=MappingProxyType(records)) + return cls(projection=projection, context=context) @dataclass(slots=True) @@ -32,6 +102,34 @@ class RuntimeSourceBindingContextCache: tuple[int, tuple[str, ...], object, int | None], "SourceUniverseRuntimeState", ] = field(default_factory=dict) + source_resolution_snapshots: dict[ + tuple[int, int, tuple[Hashable, ...], tuple[MetadataExtractionRule, ...]], + RuntimeSourceResolutionSnapshot, + ] = field(default_factory=dict) + + def source_pattern_context( + self, + *, + parser: "FilenameParser", + projection: "VirtualWorkspaceSourceProjection", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> "SourcePatternResolutionContext": + """Own one normalized runtime snapshot for exact source declarations.""" + key = (id(projection), id(parser), parser.semantic_identity(), metadata_rules) + cached = self.source_resolution_snapshots.get(key) + if cached is not None: + return cached.context + snapshot = RuntimeSourceResolutionSnapshot.from_projection( + parser=parser, + projection=projection, + metadata_rules=metadata_rules, + ) + self.source_resolution_snapshots[key] = snapshot + return snapshot.context + + def __reduce__(self) -> tuple[type[RuntimeSourceBindingContextCache], tuple[()]]: + """Transport reconstructs all derived caches from declaration defaults.""" + return type(self), () def normalized_source_metadata( self, diff --git a/openhcs/core/runtime_spatial_graph.py b/openhcs/core/runtime_spatial_graph.py index f12cc3abaf..db26f65412 100644 --- a/openhcs/core/runtime_spatial_graph.py +++ b/openhcs/core/runtime_spatial_graph.py @@ -201,9 +201,7 @@ class SpatialGraph(SourceImageProvenanceFields, NamedArtifactPayload): source_plane_index: int | None = None def __post_init__(self, *source_provenance_values: object) -> None: - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_source_provenance_fields() self.validate_artifact_name() nodes = tuple(self.nodes) diff --git a/openhcs/core/runtime_stack_cache.py b/openhcs/core/runtime_stack_cache.py index 4b92cb0020..6c2248777b 100644 --- a/openhcs/core/runtime_stack_cache.py +++ b/openhcs/core/runtime_stack_cache.py @@ -27,18 +27,11 @@ def from_paths( return cls(tuple(str(path) for path in paths), str(memory_type)) -@dataclass(frozen=True, slots=True) -class RuntimeImageStackCacheValue: - """Cached payload for an ordered runtime-slice path set.""" - - stack: RuntimeArrayData - - @dataclass(slots=True) class RuntimeImageStackCache: """Processing-context-local cache for adjacent step stack reuse.""" - stacks: dict[RuntimeImageStackCacheKey, RuntimeImageStackCacheValue] = field( + stacks: dict[RuntimeImageStackCacheKey, RuntimeArrayData] = field( default_factory=dict ) @@ -47,7 +40,7 @@ def get( paths: tuple[str, ...], *, memory_type: str, - ) -> RuntimeImageStackCacheValue | None: + ) -> RuntimeArrayData | None: """Return a cached stack for the exact ordered path and memory contract.""" return self.stacks.get( RuntimeImageStackCacheKey.from_paths(paths, memory_type=memory_type) @@ -63,7 +56,7 @@ def store( """Store a stack produced for a just-saved ordered path set.""" self.stacks[ RuntimeImageStackCacheKey.from_paths(paths, memory_type=memory_type) - ] = RuntimeImageStackCacheValue(stack=stack) + ] = stack def discard_paths(self, paths: tuple[str, ...]) -> None: """Discard cached stacks that include any of the supplied paths.""" diff --git a/openhcs/core/source_binding_selection.py b/openhcs/core/source_binding_selection.py index 90c686c60a..329813c31f 100644 --- a/openhcs/core/source_binding_selection.py +++ b/openhcs/core/source_binding_selection.py @@ -3,7 +3,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Iterator from dataclasses import dataclass, field, replace from enum import Enum from functools import lru_cache @@ -33,8 +32,9 @@ SourceImageProvenance, ) from openhcs.core.source_metadata import ( + SourceMetadataFields, SourceMetadataMapping, - SourceMetadataRoleView, + SourceMetadataRecord, SourceMetadataValue, ) from openhcs.core.source_matching import ( @@ -77,27 +77,48 @@ def _cached_source_candidate_pattern_keys(pattern_path: str) -> tuple[str, ...]: return tuple(dict.fromkeys((pattern_path, path.as_posix(), path.name))) -@dataclass(frozen=True, slots=True) -class SourceMetadataRecord(Mapping[str, SourceMetadataValue]): - """Normalized source metadata carried across source-binding selection.""" - - fields: tuple[tuple[str, SourceMetadataValue], ...] +@dataclass(frozen=True, slots=True, eq=False) +class DeclaredSourceMetadataRecord(SourceMetadataRecord): + """Live declared metadata that still requires path-specific fallbacks.""" @classmethod - def from_mapping(cls, metadata: SourceMetadataMapping) -> "SourceMetadataRecord": + def from_mapping( + cls, metadata: SourceMetadataMapping + ) -> "DeclaredSourceMetadataRecord": return cls(tuple((str(key), value) for key, value in metadata.items())) - def __getitem__(self, key: str) -> SourceMetadataValue: - for field_key, value in self.fields: - if field_key == key: - return value - raise KeyError(key) - - def __iter__(self) -> Iterator[str]: - return (key for key, _value in self.fields) - - def __len__(self) -> int: - return len(self.fields) + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> "SourceMetadataRecord | None": + """Fill parser and extraction-rule fields absent from declarations.""" + metadata: dict[str, SourceMetadataValue] = {} + merge_source_metadata(metadata, self, path=path) + parsed_metadata = parser.parse_filename(path) + if parsed_metadata is not None: + merge_source_metadata( + metadata, + { + key: value + for key, value in parsed_metadata.wire_mapping().items() + if key not in metadata + }, + path=path, + ) + rule_metadata = metadata_from_rules(path, metadata_rules) + if rule_metadata: + merge_source_metadata( + metadata, + { + key: value + for key, value in rule_metadata.items() + if key not in metadata + }, + path=path, + ) + return self.from_mapping(metadata) if metadata else None @dataclass(frozen=True, slots=True) @@ -176,7 +197,7 @@ def from_sources( str(path): ( metadata if isinstance(metadata, SourceMetadataRecord) - else SourceMetadataRecord.from_mapping(metadata) + else DeclaredSourceMetadataRecord.from_mapping(metadata) ) for path, metadata in source_metadata_by_path.items() } @@ -244,9 +265,9 @@ def candidate_filter_paths(self, pattern: SourceCandidatePath) -> tuple[str, ... dict.fromkeys( source_filter_path for metadata in self.metadata_for_paths(resolution.metadata_paths()) - for source_filter_path in SourceMetadataRoleView( + for source_filter_path in SourceMetadataFields.source_filter_paths( metadata - ).source_filter_paths() + ) ) ) if source_filter_paths: @@ -404,35 +425,10 @@ def metadata_for_paths( ) def metadata_for_path(self, path: str) -> SourceMetadataRecord | None: - metadata: dict[str, SourceMetadataValue] = {} - declared_metadata = self.source_metadata_by_path.get(path) - if declared_metadata is not None: - merge_source_metadata(metadata, declared_metadata, path=path) - parsed_metadata = self.parser.parse_filename(path) - if parsed_metadata is not None: - merge_source_metadata( - metadata, - { - key: value - for key, value in parsed_metadata.wire_mapping().items() - if key not in metadata - }, - path=path, - ) - rule_metadata = metadata_from_rules(path, self.metadata_rules) - if rule_metadata: - merge_source_metadata( - metadata, - { - key: value - for key, value in rule_metadata.items() - if key not in metadata - }, - path=path, - ) - if metadata: - return SourceMetadataRecord.from_mapping(metadata) - return None + record = self.source_metadata_by_path.get(path) + if record is None: + record = DeclaredSourceMetadataRecord(()) + return record.resolve(path, self.parser, self.metadata_rules) def merged_metadata_for_paths( self, @@ -445,7 +441,7 @@ def merged_metadata_for_paths( if path_metadata is not None: merge_source_metadata(metadata, path_metadata, path=path) if metadata: - return SourceMetadataRecord.from_mapping(metadata) + return DeclaredSourceMetadataRecord.from_mapping(metadata) return None def source_metadata_by_paths( @@ -1369,7 +1365,7 @@ def _source_image_set_identity( metadata = ( metadata_candidates.values[0] if metadata_candidates.values - else SourceMetadataRecord(()) + else DeclaredSourceMetadataRecord(()) ) return SourceImageSetIdentity.from_metadata( metadata, @@ -1708,7 +1704,7 @@ def source_metadata_by_path(self) -> Mapping[str, SourceMetadataMapping]: def source_context(self) -> SourcePatternResolutionContext: metadata_rules = self.plan.source_binding_plan.metadata_rules if self.source_projection is not None: - return SourcePatternResolutionContext.from_projection( + return self.context.runtime_source_binding_context_cache.source_pattern_context( parser=self.context.microscope_handler.parser, projection=self.source_projection, metadata_rules=metadata_rules, diff --git a/openhcs/core/source_binding_workspace.py b/openhcs/core/source_binding_workspace.py index 02230380d8..3567219876 100644 --- a/openhcs/core/source_binding_workspace.py +++ b/openhcs/core/source_binding_workspace.py @@ -45,7 +45,7 @@ OriginalSourceMetadata, SourceFilterPathMetadata, SourceMetadataMapping, - SourceMetadataRoleView, + SourceMetadataFields, SourceMetadataScalar, SourceVoxelSpacing, SourceComponentProjectionStrategy, @@ -1373,7 +1373,7 @@ def _shared_candidate_metadata( value_sets_by_key: dict[str, set[object]] = {} counts_by_key: dict[str, int] = {} for candidate in candidates: - for key, value in SourceMetadataRoleView(candidate.metadata).scalar_items(): + for key, value in SourceMetadataFields.scalar_items(candidate.metadata): value_sets_by_key.setdefault(key, set()).add(value) counts_by_key[key] = counts_by_key.get(key, 0) + 1 candidate_count = len(candidates) diff --git a/openhcs/core/source_bindings.py b/openhcs/core/source_bindings.py index 7f2bf54e30..9b00e410dd 100644 --- a/openhcs/core/source_bindings.py +++ b/openhcs/core/source_bindings.py @@ -38,9 +38,9 @@ from openhcs.core.runtime_tabular_values import FieldSpec from openhcs.core.source_metadata import ( SourceMetadataIdentityItems, - SourceMetadataIdentityProjection, + SourceMetadataFields, + ResolvedSourceMetadataRecord, SourceMetadataMapping, - SourceMetadataRoleView, SourceMetadataScalar, SourceMetadataValue, SourceVoxelSpacing, @@ -2000,7 +2000,7 @@ def metadata_fields_for_realized_source_metadata( declared_names = frozenset(field.name for field in declared_fields) values_by_name: dict[str, list[SourceMetadataScalar]] = {} for metadata in realized_source_metadata: - for field_name, value in SourceMetadataRoleView(metadata).original_items(): + for field_name, value in SourceMetadataFields.original_items(metadata): if ( field_name not in declared_names and field_name not in excluded_names @@ -2360,31 +2360,13 @@ class SourceBindingRuntimeMetadataNormalizer: def normalized(self) -> Mapping[str, SourceMetadataMapping]: return MappingProxyType( { - str(path): MappingProxyType( - { - str(key): self.normalized_value(value) - for key, value in source_metadata_dict(metadata).items() - } + str(path): ResolvedSourceMetadataRecord.normalized_mapping( + source_metadata_dict(metadata) ) for path, metadata in self.source_metadata_by_path.items() } ) - @classmethod - def normalized_value(cls, value: SourceMetadataValue) -> SourceMetadataValue: - if isinstance(value, Mapping): - return MappingProxyType( - { - str(key): cls.normalized_scalar(nested_value) - for key, nested_value in value.items() - } - ) - return cls.normalized_scalar(value) - - @staticmethod - def normalized_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: - return source_metadata_scalar(value) - @dataclass(frozen=True) class SourceBindingRuntimeContext: @@ -2538,7 +2520,7 @@ def source_metadata_identity( cached = self._source_metadata_identity if cached is None: cached = tuple( - (path, SourceMetadataIdentityProjection(metadata).items()) + (path, SourceMetadataFields.identity_items(metadata)) for path, metadata in sorted(self.source_metadata_by_path.items()) ) object.__setattr__(self, "_source_metadata_identity", cached) @@ -2673,9 +2655,9 @@ def metadata_identity_for_paths( identity: list[tuple[str, SourceMetadataIdentityItems]] = [] for path in paths: if path in self.source_metadata_by_path: - metadata = SourceMetadataIdentityProjection( + metadata = SourceMetadataFields.identity_items( self.source_metadata_by_path[path] - ).items() + ) else: metadata = () identity.append((path, metadata)) diff --git a/openhcs/core/source_image_provenance.py b/openhcs/core/source_image_provenance.py index 71ff6ef174..025d77b87f 100644 --- a/openhcs/core/source_image_provenance.py +++ b/openhcs/core/source_image_provenance.py @@ -20,6 +20,8 @@ from openhcs.core.source_metadata import ( SOURCE_PLANE_COUNT_FIELD, SOURCE_PLANE_INDEX_FIELD, + OwnedSourceMetadataFields, + SourceMetadataFields, SourceMetadataMapping, SourceMetadataScalar, SourceMetadataValue, @@ -69,20 +71,12 @@ def normalize_source_path(source_path: str | None) -> str | None: return str(source_path) -def _normalize_component_metadata( - metadata: SourceComponentMetadata | None, -) -> SourceComponentMetadata | None: - if metadata is None: - return None - return MappingProxyType(dict(metadata)) - - def _component_metadata_identity( metadata: SourceComponentMetadata | None, ) -> tuple[tuple[str, str], ...] | None: if metadata is None: return None - return tuple(sorted((str(key), repr(value)) for key, value in metadata.items())) + return SourceMetadataFields.provenance_identity_items(metadata) def _tuple_value(values: tuple[Any, ...], index: int) -> Any | None: @@ -101,7 +95,10 @@ class SourceImageIdentity: def __post_init__(self) -> None: self.path = normalize_source_path(self.path) - self.component_metadata = _normalize_component_metadata(self.component_metadata) + if self.component_metadata is not None: + self.component_metadata = SourceMetadataFields.readonly_snapshot( + self.component_metadata + ) self._identity = ( self.path, _component_metadata_identity(self.component_metadata), @@ -147,21 +144,10 @@ def component_metadata_with_missing_from( if fallback.component_metadata is None: return self.component_metadata - merged = dict(self.component_metadata) - for component in AllComponents: - if source_component_metadata_value(merged, component) is not None: - continue - fallback_value = source_component_metadata_raw_value( - fallback.component_metadata, - component, - ) - if fallback_value is not None: - merged[component.value] = fallback_value - if self.filename_extension is None: - extension = fallback.filename_extension - if extension is not None: - merged["extension"] = extension - return MappingProxyType(merged) + merged = SourceMetadataFields.with_missing_from( + self.component_metadata, fallback.component_metadata + ) + return SourceMetadataFields.readonly_snapshot(merged) def with_parsed_path_components( self, @@ -1505,18 +1491,15 @@ def component_metadata_for_plane( self, plane_index: int, ) -> SourceComponentMetadata: - metadata = { - **dict(self.scalar_metadata), - SOURCE_PLANE_INDEX_FIELD: str(plane_index), - SOURCE_PLANE_COUNT_FIELD: str(self.source_plane_count), - } - return MappingProxyType( - with_source_component_metadata( - metadata, - AllComponents.Z_INDEX, - self.z_index_for_plane(plane_index), - ) + metadata = SourceMetadataFields.with_fields( + SourceMetadataFields.composition_snapshot(self.scalar_metadata), + { + SOURCE_PLANE_INDEX_FIELD: str(plane_index), + SOURCE_PLANE_COUNT_FIELD: str(self.source_plane_count), + }, + components=((AllComponents.Z_INDEX, self.z_index_for_plane(plane_index)),), ) + return SourceMetadataFields.readonly_snapshot(metadata) def z_index_for_plane(self, plane_index: int) -> int: scalar_z_index = source_component_metadata_value( @@ -1585,7 +1568,9 @@ def common_source_component_metadata( } if not common_metadata: return None - return MappingProxyType(common_metadata) + return SourceMetadataFields.readonly_snapshot( + SourceMetadataFields.derived_mapping(consensus, common_metadata) + ) def source_component_metadata_consensus( @@ -1594,7 +1579,9 @@ def source_component_metadata_consensus( """Return every metadata field, nulling values that differ between planes.""" metadata_values = tuple( - dict(metadata) for metadata in metadata_by_plane if metadata is not None + SourceMetadataFields.composition_snapshot(metadata) + for metadata in metadata_by_plane + if metadata is not None ) if len(metadata_values) != len(metadata_by_plane): return None @@ -1618,7 +1605,17 @@ def source_component_metadata_consensus( and all(value == present_values[0] for value in present_values) else None ) - return MappingProxyType(consensus) + owner = ( + metadata_values[0] + if all( + isinstance(metadata, OwnedSourceMetadataFields) + for metadata in metadata_values + ) + else {} + ) + return SourceMetadataFields.readonly_snapshot( + SourceMetadataFields.derived_mapping(owner, consensus) + ) def common_source_path(paths: Sequence[str | None]) -> str | None: @@ -1775,8 +1772,17 @@ class SourceImageProvenanceFields: def absorb_explicit_source_provenance( self, - explicit: SourceImageProvenance, + values: SourceProvenanceInitValues, ) -> None: + """Decode authored aliases once, leaving absent constructor facts absent.""" + if all( + value is default + for value, default in zip( + values, SourceImageProvenance.__init__.__defaults__, strict=True + ) + ): + return + explicit = SourceImageProvenance.from_init_values(values) if not explicit.has_values: return self.source_provenance = explicit.with_missing_from(self.source_provenance) diff --git a/openhcs/core/source_matching.py b/openhcs/core/source_matching.py index 3c9d1e0d86..66108ec9fe 100644 --- a/openhcs/core/source_matching.py +++ b/openhcs/core/source_matching.py @@ -30,12 +30,13 @@ OriginalSourceMetadata, SourceFilterPathMetadata, SourceMetadataMapping, - SourceMetadataRoleView, + SourceMetadataFields, SourceMetadataScalar, SourceMetadataValue, SourceComponentProjectionStrategy, path_metadata_values_equivalent, source_metadata_field_identity, + source_metadata_component, source_metadata_dict, source_metadata_scalar, ) @@ -709,7 +710,7 @@ def overlay_source_metadata( for component, value in projected_components.items(): overlaid = with_source_component_metadata(overlaid, component, value) - for field, value in SourceMetadataRoleView(additions).scalar_items(): + for field, value in SourceMetadataFields.scalar_items(additions): if value is None: continue component = source_metadata_component(field) @@ -770,7 +771,7 @@ def source_metadata_value( key: str, ) -> SourceMetadataScalar: """Return one source-literal metadata value by its exact declared key.""" - return _source_metadata_lookup_projection(metadata).value(key) + return SourceMetadataFields.literal_value(metadata, key) def source_component_metadata_value( @@ -819,7 +820,7 @@ def semantic_source_metadata_value( field_identity = source_metadata_field_identity(field_name) semantic_values = tuple( value - for field, value in SourceMetadataRoleView(metadata).scalar_items() + for field, value in SourceMetadataFields.scalar_items(metadata) if value is not None and source_metadata_field_identity(str(field)) == field_identity ) @@ -846,14 +847,14 @@ def source_component_metadata_values( component: AllComponents, ) -> tuple[str, ...]: """Return all metadata values that semantically describe a component.""" - return _source_metadata_lookup_projection(metadata).component_values(component) + return SourceMetadataFields.component_values(metadata, component) def with_source_component_metadata( metadata: SourceMetadataMapping, component: AllComponents, value: SourceMetadataScalar, -) -> dict[str, SourceMetadataValue]: +) -> SourceMetadataMapping: """Return metadata with one canonical component value. Source schemas can carry CellProfiler spellings such as ``Well`` and OpenHCS @@ -861,15 +862,7 @@ def with_source_component_metadata( every semantic spelling so later normalized lookups cannot observe both the old and new component values. """ - return { - **{ - key: field_value - for key, field_value in metadata.items() - if key == ORIGINAL_SOURCE_METADATA_FIELD - or source_metadata_component(str(key)) is not component - }, - component.value: str(value), - } + return SourceMetadataFields.with_component(metadata, component, value) def source_metadata_values_equal( @@ -969,62 +962,10 @@ def constraint_matches_metadata( return any( metadata_value is not None and source_metadata_values_equal(str(metadata_value), value) - for metadata_value in SourceMetadataRoleView(metadata).scalar_values() + for metadata_value in SourceMetadataFields.scalar_values(metadata) ) -@lru_cache(maxsize=256) -def source_metadata_component(field: str) -> AllComponents | None: - """Return the nominal component owner of a metadata field.""" - - return SourceComponentProjectionStrategy.component_for_metadata_field(field) - - -@dataclass(frozen=True, slots=True) -class SourceMetadataLookupProjection: - """Lookup projection over the current contents of one source metadata mapping.""" - - scalar_items: tuple[tuple[str, SourceMetadataScalar], ...] - original_items: tuple[tuple[str, SourceMetadataScalar], ...] - - def value(self, key: str) -> SourceMetadataScalar: - """Return a source-literal metadata value by exact key.""" - for candidate_key, value in self.original_items: - if str(candidate_key) == key and value is not None: - return value - for candidate_key, value in self.scalar_items: - if str(candidate_key) == key and value is not None: - return value - return None - - def component_values(self, component: AllComponents) -> tuple[str, ...]: - """Return all metadata values that semantically describe a component.""" - values: list[str] = [] - for field, field_value in self.scalar_items: - field_text = str(field) - if field_text == component.value and field_value is not None: - values.append(str(field_value)) - for field, field_value in self.scalar_items: - field_text = str(field) - if ( - field_text != component.value - and source_metadata_component(field_text) is component - ) and field_value is not None: - values.append(str(field_value)) - return tuple(dict.fromkeys(values)) - - -def _source_metadata_lookup_projection( - metadata: SourceMetadataMapping, -) -> SourceMetadataLookupProjection: - """Derive a lookup projection from the current nominal metadata role view.""" - role_view = SourceMetadataRoleView(metadata) - return SourceMetadataLookupProjection( - scalar_items=role_view.scalar_items(), - original_items=role_view.original_items(), - ) - - def _require_filter_value(clause: SourceFilterClause) -> str: if clause.value is None: raise ValueError( diff --git a/openhcs/core/source_metadata.py b/openhcs/core/source_metadata.py index 63c7e4e049..06fc651d4a 100644 --- a/openhcs/core/source_metadata.py +++ b/openhcs/core/source_metadata.py @@ -3,13 +3,14 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, field from enum import Enum from functools import lru_cache from math import isfinite from pathlib import Path -from typing import ClassVar, TypeAlias +from types import MappingProxyType +from typing import ClassVar, NoReturn, Self, TYPE_CHECKING, TypeAlias, TypeVar from metaclass_registry import AutoRegisterMeta from zmqruntime.viewer_protocol import ViewerWireField @@ -17,6 +18,11 @@ from openhcs.constants.constants import AllComponents from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +if TYPE_CHECKING: + from openhcs.core.source_bindings import MetadataExtractionRule + from openhcs.microscopes.microscope_interfaces import FilenameParser + + ORIGINAL_SOURCE_METADATA_FIELD = "OpenHCSOriginalSourceMetadata" SOURCE_FILTER_PATHS_METADATA_FIELD = "OpenHCSSourceFilterPaths" SOURCE_PLANE_INDEX_FIELD = "source_plane_index" @@ -24,7 +30,8 @@ SOURCE_VOXEL_SPACING_FIELD = "OpenHCSSourceVoxelSpacingZYX" SOURCE_VOXEL_SPACING_UNIT_FIELD = "OpenHCSSourceVoxelSpacingUnit" -SourceMetadataScalar: TypeAlias = str | int | float | bool | None +SourceMetadataNonNullScalar: TypeAlias = str | int | float | bool +SourceMetadataScalar: TypeAlias = SourceMetadataNonNullScalar | None SourceMetadataValue: TypeAlias = ( SourceMetadataScalar | Mapping[str, SourceMetadataScalar] ) @@ -38,6 +45,500 @@ ] +_MetadataViewT = TypeVar("_MetadataViewT") + + +@dataclass(frozen=True, slots=True, eq=False) +class SourceMetadataFields(Mapping[str, SourceMetadataValue]): + """Common field projection algorithms with nominal lifetime-selected views.""" + + fields: tuple[tuple[str, SourceMetadataValue], ...] + + @classmethod + def from_mapping(cls, metadata: SourceMetadataMapping) -> Self: + return cls(tuple(metadata.items())) + + @classmethod + def normalized_value(cls, value: SourceMetadataValue) -> SourceMetadataValue: + if isinstance(value, Mapping): + return MappingProxyType( + { + str(key): cls.normalized_scalar(nested_value) + for key, nested_value in value.items() + } + ) + return cls.normalized_scalar(value) + + @classmethod + def normalized_mapping( + cls, metadata: SourceMetadataMapping + ) -> SourceMetadataMapping: + return MappingProxyType( + {str(key): cls.normalized_value(value) for key, value in metadata.items()} + ) + + @classmethod + def normalized_scalar(cls, value: SourceMetadataScalar) -> SourceMetadataScalar: + """Admit the shared scalar grammar before applying the owner's policy.""" + if value is None: + return None + if isinstance(value, SourceMetadataNonNullScalar): + return cls._normalize_admitted_scalar(value) + return cls._reject_scalar(value) + + @staticmethod + def _normalize_admitted_scalar( + value: SourceMetadataNonNullScalar, + ) -> SourceMetadataNonNullScalar: + if isinstance(value, str): + return canonical_path_metadata_value(value) + return value + + @staticmethod + def _reject_scalar(value: object) -> NoReturn: + raise TypeError( + "Source metadata scalar values must be str, int, float, bool, or None, " + f"got {type(value).__name__}." + ) + + def __getitem__(self, key: str) -> SourceMetadataValue: + for field_key, value in self.fields: + if field_key == key: + return value + raise KeyError(key) + + def __iter__(self) -> Iterator[str]: + return (key for key, _value in self.fields) + + def __len__(self) -> int: + return len(self.fields) + + @abstractmethod + def __eq__(self, other: object) -> bool: + """Compare through the declared record or mapping namespace.""" + + def _derived_view( + self, key: str | AllComponents, derive: Callable[[], _MetadataViewT] + ) -> _MetadataViewT: + return derive() + + @staticmethod + def _view( + metadata: SourceMetadataMapping, + key: str | AllComponents, + derive: Callable[[], _MetadataViewT], + ) -> _MetadataViewT: + if isinstance(metadata, SourceMetadataFields): + return metadata._derived_view(key, derive) + return derive() + + @classmethod + def scalar_items( + cls, metadata: SourceMetadataMapping + ) -> tuple[tuple[str, SourceMetadataScalar], ...]: + """Read current raw fields or the immutable owner's scalar projection.""" + return cls._view( + metadata, + "scalar_items", + lambda: tuple( + (str(key), value) + for key, value in metadata.items() + if key != ORIGINAL_SOURCE_METADATA_FIELD + and not isinstance(value, Mapping) + ), + ) + + @classmethod + def scalar_values( + cls, metadata: SourceMetadataMapping + ) -> tuple[SourceMetadataScalar, ...]: + return tuple(value for _key, value in cls.scalar_items(metadata)) + + @classmethod + def original_items( + cls, metadata: SourceMetadataMapping + ) -> tuple[tuple[str, SourceMetadataScalar], ...]: + def project() -> tuple[tuple[str, SourceMetadataScalar], ...]: + original_metadata = metadata.get(ORIGINAL_SOURCE_METADATA_FIELD) + if original_metadata is None: + return () + return OriginalSourceMetadata.from_reserved_value( + original_metadata, path=ORIGINAL_SOURCE_METADATA_FIELD + ).fields + + return cls._view(metadata, "original_items", project) + + @classmethod + def source_filter_paths(cls, metadata: SourceMetadataMapping) -> tuple[str, ...]: + def project() -> tuple[str, ...]: + paths = metadata.get(SOURCE_FILTER_PATHS_METADATA_FIELD) + if paths is None: + return () + return SourceFilterPathMetadata.from_reserved_value( + paths, path=SOURCE_FILTER_PATHS_METADATA_FIELD + ).paths + + return cls._view(metadata, "source_filter_paths", project) + + @classmethod + def identity_items( + cls, metadata: SourceMetadataMapping + ) -> SourceMetadataIdentityItems: + def project() -> SourceMetadataIdentityItems: + return tuple( + sorted( + ( + str(key), + ( + tuple(sorted((str(k), v) for k, v in value.items())) + if isinstance(value, Mapping) + else value + ), + ) + for key, value in metadata.items() + ) + ) + + return cls._view(metadata, "identity_items", project) + + @classmethod + def provenance_identity_items( + cls, metadata: SourceMetadataMapping + ) -> tuple[tuple[str, str], ...]: + """Derive stable provenance identity from the owner's canonical field view.""" + return cls._view( + metadata, + "provenance_identity_items", + lambda: tuple((key, repr(value)) for key, value in cls.identity_items(metadata)), + ) + + @classmethod + def literal_value( + cls, metadata: SourceMetadataMapping, key: str + ) -> SourceMetadataScalar: + # Preserve the existing lookup's role-validation order even when the + # requested literal could be read without the original metadata role. + scalars = cls.scalar_items(metadata) + original = cls.original_items(metadata) + for fields in (original, scalars): + for candidate_key, value in fields: + if str(candidate_key) == key and value is not None: + return value + return None + + @classmethod + def component_value( + cls, metadata: SourceMetadataMapping, component: AllComponents + ) -> str | None: + return cls._view( + metadata, + component, + lambda: SourceComponentProjectionStrategy.for_enum_member( + component + ).metadata_value(metadata), + ) + + @classmethod + def component_values( + cls, metadata: SourceMetadataMapping, component: AllComponents + ) -> tuple[str, ...]: + scalars = cls.scalar_items(metadata) + cls.original_items(metadata) + values = [ + str(value) + for field, value in scalars + if str(field) == component.value and value is not None + ] + values.extend( + str(value) + for field, value in scalars + if str(field) != component.value + and source_metadata_component(str(field)) is component + and value is not None + ) + return tuple(dict.fromkeys(values)) + + @staticmethod + def readonly_snapshot(metadata: SourceMetadataMapping) -> SourceMetadataMapping: + """Preserve the old mapping snapshot while retaining an owned record's lifetime.""" + if isinstance(metadata, SourceMetadataFields): + return metadata._readonly_snapshot() + return MappingProxyType(dict(metadata)) + + def _readonly_snapshot(self) -> SourceMetadataMapping: + return MappingProxyType(dict(self)) + + @staticmethod + def composition_snapshot(metadata: SourceMetadataMapping) -> SourceMetadataMapping: + """Capture one composition's raw fields; owned records already hold that snapshot.""" + if isinstance(metadata, SourceMetadataFields): + return metadata._composition_snapshot() + return dict(metadata) + + def _composition_snapshot(self) -> SourceMetadataMapping: + return dict(self) + + @staticmethod + def derived_mapping( + metadata: SourceMetadataMapping, + fields: dict[str, SourceMetadataValue] | None, + ) -> SourceMetadataMapping: + """Keep owned lifetime on derived fields; raw callers receive a fresh dict.""" + if isinstance(metadata, SourceMetadataFields): + return metadata._derive_fields(fields) + return dict(metadata) if fields is None else fields + + def _derive_fields( + self, fields: dict[str, SourceMetadataValue] | None + ) -> SourceMetadataMapping: + return dict(self) if fields is None else fields + + @classmethod + def with_fields( + cls, + metadata: SourceMetadataMapping, + updates: SourceMetadataMapping, + *, + without: Iterable[str] = (), + components: Iterable[tuple[AllComponents, SourceMetadataScalar]] = (), + after_components: SourceMetadataMapping | None = None, + ) -> SourceMetadataMapping: + excluded = frozenset(without) + fields = {key: value for key, value in metadata.items() if key not in excluded} + fields.update(updates) + for component, value in components: + fields = { + key: field_value + for key, field_value in fields.items() + if key == ORIGINAL_SOURCE_METADATA_FIELD + or source_metadata_component(str(key)) is not component + } + fields[component.value] = str(value) + if after_components is not None: + fields.update(after_components) + return cls.derived_mapping(metadata, fields) + + @classmethod + def with_component( + cls, + metadata: SourceMetadataMapping, + component: AllComponents, + value: SourceMetadataScalar, + ) -> SourceMetadataMapping: + return cls.with_fields(metadata, {}, components=((component, value),)) + + @classmethod + def with_missing_from( + cls, metadata: SourceMetadataMapping, fallback: SourceMetadataMapping + ) -> SourceMetadataMapping: + merged: dict[str, SourceMetadataValue] | None = None + current = metadata + for component in AllComponents: + if cls.component_value(current, component) is not None: + continue + value = cls.component_value(fallback, component) + if value is not None: + if merged is None: + merged = dict(metadata) + merged[component.value] = value + current = merged + if cls.literal_value(metadata, "extension") is None: + extension = cls.literal_value(fallback, "extension") + if extension is not None: + if merged is None: + merged = dict(metadata) + merged["extension"] = str(extension) + if merged is None: + return cls.derived_mapping(metadata, None) + return cls.derived_mapping(metadata, merged) + + +@dataclass(frozen=True, slots=True, eq=False) +class SourceMetadataRecord(SourceMetadataFields): + """Ordered source metadata carried through declared selector resolution.""" + + @abstractmethod + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple["MetadataExtractionRule", ...], + ) -> "SourceMetadataRecord | None": + """Resolve fields through their declared metadata lifetime.""" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SourceMetadataRecord): + return NotImplemented + return self.fields == other.fields + + def __hash__(self) -> int: + return hash((self.fields,)) + + +@dataclass(frozen=True, slots=True, eq=False) +class OwnedSourceMetadataFields(SourceMetadataFields): + """Deep field ownership, indexed lookup, bounded views and local transport lifetime.""" + + _field_index: Mapping[str, SourceMetadataValue] = field( + init=False, repr=False, compare=False + ) + _views: dict[str | AllComponents, object] = field( + default_factory=dict, init=False, repr=False, compare=False + ) + _cacheable: bool = field(init=False, repr=False, compare=False) + + def __post_init__(self) -> None: + fields = tuple( + (str(key), self.normalized_value(value)) for key, value in self.fields + ) + object.__setattr__(self, "fields", self._normalized_fields(fields)) + self._initialize_derived_views() + + @staticmethod + def _normalized_fields( + fields: tuple[tuple[str, SourceMetadataValue], ...], + ) -> tuple[tuple[str, SourceMetadataValue], ...]: + return fields + + def _initialize_derived_views(self) -> None: + object.__setattr__(self, "_views", {}) + index: dict[str, SourceMetadataValue] = {} + for key, value in self.fields: + index.setdefault(key, value) + object.__setattr__(self, "_field_index", MappingProxyType(index)) + object.__setattr__( + self, + "_cacheable", + all( + ( + all( + type(v) in (str, int, float, bool, type(None)) + for v in value.values() + ) + if isinstance(value, Mapping) + else type(value) in (str, int, float, bool, type(None)) + ) + for _key, value in self.fields + ), + ) + + def __reduce__(self): + fields = tuple( + (key, dict(value) if isinstance(value, Mapping) else value) + for key, value in self.fields + ) + return (type(self)._from_serialized_fields, (fields,)) + + @classmethod + def _from_serialized_fields( + cls, fields: tuple[tuple[str, SourceMetadataValue], ...] + ) -> Self: + """Restore stored spelling and ownership without serializing derived views.""" + record = cls.__new__(cls) + object.__setattr__( + record, + "fields", + tuple( + ( + key, + ( + MappingProxyType(dict(value)) + if isinstance(value, Mapping) + else value + ), + ) + for key, value in fields + ), + ) + record._initialize_derived_views() + return record + + def __getitem__(self, key: str) -> SourceMetadataValue: + return self._field_index[key] + + @property + def metadata_contents(self) -> SourceMetadataMapping: + """Read-only mapping contents, separate from ordered record equality.""" + return self._field_index + + def _derived_view( + self, key: str | AllComponents, derive: Callable[[], _MetadataViewT] + ) -> _MetadataViewT: + if not self._cacheable: + return derive() + if key not in self._views: + self._views[key] = derive() + return self._views[key] + + def _readonly_snapshot(self) -> SourceMetadataMapping: + return self + + def _composition_snapshot(self) -> SourceMetadataMapping: + return self + + def _derive_fields( + self, fields: dict[str, SourceMetadataValue] | None + ) -> SourceMetadataMapping: + return self if fields is None else DurableSourceMetadata.from_mapping(fields) + + +@dataclass(frozen=True, slots=True, eq=False) +class ResolvedSourceMetadataRecord(OwnedSourceMetadataFields, SourceMetadataRecord): + """Owned runtime fields with the existing ordered selector record contract.""" + + def _readonly_snapshot(self) -> SourceMetadataMapping: + return self._derived_view( + "readonly_snapshot", + lambda: DurableSourceMetadata.from_mapping(self._field_index), + ) + + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple["MetadataExtractionRule", ...], + ) -> SourceMetadataRecord | None: + return self if self.fields else None + + +@dataclass(frozen=True, slots=True, eq=False) +class DurableSourceMetadata(OwnedSourceMetadataFields): + """Owned literal metadata from durable storage and image field derivation.""" + + @staticmethod + def _normalized_fields( + fields: tuple[tuple[str, SourceMetadataValue], ...], + ) -> tuple[tuple[str, SourceMetadataValue], ...]: + return tuple(dict(fields).items()) + + def __eq__(self, other: object) -> bool: + if isinstance(other, SourceMetadataRecord): + return NotImplemented + return Mapping.__eq__(self, other) + + __hash__ = None + + @staticmethod + def _normalize_admitted_scalar( + value: SourceMetadataNonNullScalar, + ) -> SourceMetadataNonNullScalar: + return value + + @staticmethod + def _reject_scalar(value: object) -> NoReturn: + if isinstance(value, Mapping) or ( + isinstance(value, Sequence) and not isinstance(value, str) + ): + raise RuntimeError( + "virtual_workspace source metadata supports scalar values and " + "one-level scalar mappings only." + ) + raise RuntimeError( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + + def source_metadata_dict( metadata: SourceMetadataMapping, ) -> dict[str, SourceMetadataValue]: @@ -69,16 +570,7 @@ def source_metadata_dict( def source_metadata_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: """Return the canonical scalar representation stored in source metadata.""" - if value is None: - return None - if not isinstance(value, (str, int, float, bool)): - raise TypeError( - "Source metadata scalar values must be str, int, float, bool, or None, " - f"got {type(value).__name__}." - ) - if isinstance(value, str): - return canonical_path_metadata_value(value) - return value + return SourceMetadataFields.normalized_scalar(value) def canonical_path_metadata_value(value: str) -> str: @@ -321,6 +813,13 @@ def __post_init__(self) -> None: if not isinstance(self.unit, SourceVoxelSpacingUnit): raise TypeError("SourceVoxelSpacing.unit must be SourceVoxelSpacingUnit.") + @classmethod + def coerce( + cls, value: "SourceVoxelSpacing | Sequence[float]" + ) -> "SourceVoxelSpacing": + """Retain nominal spacing or validate authored coordinate values.""" + return value if isinstance(value, cls) else cls(tuple(value)) + @property def has_values(self) -> bool: return bool(self.values_zyx) @@ -483,72 +982,7 @@ class SourceVoxelSpacingFields: ) def normalize_source_voxel_spacing_fields(self) -> None: - if not isinstance(self.source_voxel_spacing, SourceVoxelSpacing): - self.source_voxel_spacing = SourceVoxelSpacing( - tuple(self.source_voxel_spacing) - ) - - -@dataclass(frozen=True, slots=True) -class SourceMetadataRoleView: - """Role-aware metadata view for literal selectors versus component axes.""" - - metadata: SourceMetadataMapping - - def scalar_items(self) -> tuple[tuple[str, SourceMetadataScalar], ...]: - items: list[tuple[str, SourceMetadataScalar]] = [] - for key, value in self.metadata.items(): - if key == ORIGINAL_SOURCE_METADATA_FIELD or isinstance(value, Mapping): - continue - items.append((str(key), value)) - return tuple(items) - - def scalar_values(self) -> tuple[SourceMetadataScalar, ...]: - return tuple(value for _key, value in self.scalar_items()) - - def original_items(self) -> tuple[tuple[str, SourceMetadataScalar], ...]: - original_metadata = self.metadata.get(ORIGINAL_SOURCE_METADATA_FIELD) - if original_metadata is None: - return () - return OriginalSourceMetadata.from_reserved_value( - original_metadata, - path=ORIGINAL_SOURCE_METADATA_FIELD, - ).fields - - def source_filter_paths(self) -> tuple[str, ...]: - source_filter_paths = self.metadata.get(SOURCE_FILTER_PATHS_METADATA_FIELD) - if source_filter_paths is None: - return () - return SourceFilterPathMetadata.from_reserved_value( - source_filter_paths, - path=SOURCE_FILTER_PATHS_METADATA_FIELD, - ).paths - - -@dataclass(frozen=True, slots=True) -class SourceMetadataIdentityProjection: - """Stable hash projection for source metadata records.""" - - metadata: SourceMetadataMapping - - def items(self) -> SourceMetadataIdentityItems: - projected: list[tuple[str, SourceMetadataIdentityValue]] = [] - for key, value in self.metadata.items(): - if isinstance(value, Mapping): - projected.append( - ( - str(key), - tuple( - sorted( - (str(nested_key), nested_value) - for nested_key, nested_value in value.items() - ) - ), - ) - ) - continue - projected.append((str(key), value)) - return tuple(sorted(projected)) + self.source_voxel_spacing = SourceVoxelSpacing.coerce(self.source_voxel_spacing) @lru_cache(maxsize=4096) @@ -565,6 +999,12 @@ def source_metadata_field_identity(field: str) -> str: ) +@lru_cache(maxsize=256) +def source_metadata_component(field: str) -> AllComponents | None: + """Return the nominal component owner of a metadata field.""" + return SourceComponentProjectionStrategy.component_for_metadata_field(field) + + class SourceComponentProjectionStrategy( EnumKeyedStrategyMixin[AllComponents], ABC, @@ -591,7 +1031,7 @@ def metadata_component( component: AllComponents, metadata: SourceMetadataMapping, ) -> str | None: - return cls.for_enum_member(component).metadata_value(metadata) + return SourceMetadataFields.component_value(metadata, component) @classmethod def component_for_metadata_field( @@ -625,7 +1065,7 @@ def _metadata_group_value( metadata: SourceMetadataMapping, group: tuple[str, ...], ) -> str | None: - scalar_items = SourceMetadataRoleView(metadata).scalar_items() + scalar_items = SourceMetadataFields.scalar_items(metadata) for alias in group: alias_identity = source_metadata_field_identity(alias) for field_name, value in scalar_items: diff --git a/openhcs/core/source_projection.py b/openhcs/core/source_projection.py index c90097091e..8711e50c62 100644 --- a/openhcs/core/source_projection.py +++ b/openhcs/core/source_projection.py @@ -35,9 +35,9 @@ ) from openhcs.core.source_metadata import ( SourceComponentProjectionStrategy, - SourceMetadataIdentityProjection, + SourceMetadataFields, + ResolvedSourceMetadataRecord, SourceMetadataMapping, - SourceMetadataRoleView, SourceMetadataValue, source_metadata_dict, source_metadata_scalar, @@ -449,7 +449,7 @@ def identity_key(self) -> tuple[object, ...]: self.declared_address, self.dataset_identity, self.store_identity, - SourceMetadataIdentityProjection(self.metadata).items(), + SourceMetadataFields.identity_items(self.metadata), ) @@ -1291,7 +1291,7 @@ def _source_metadata( metadata = source_metadata_dict(projection.source_metadata) source_component_fields = { field: value - for field, value in SourceMetadataRoleView(metadata).scalar_items() + for field, value in SourceMetadataFields.scalar_items(metadata) if ( (component := source_metadata_component(field)) is not None and field != component.value @@ -1303,7 +1303,7 @@ def _source_metadata( source_component_fields, path=projection.ref.backend_address, ) - original_metadata = dict(SourceMetadataRoleView(metadata).original_items()) + original_metadata = dict(SourceMetadataFields.original_items(metadata)) for component, value in projection.source_component_values(): canonical_value = metadata.get(component.value) conflicts_with_address = ( @@ -1375,21 +1375,6 @@ def _padded(value: str, width: int) -> str: return f"{int(value):0{width}d}" if value.isdecimal() else value -def _normalized_source_metadata_value( - value: SourceMetadataValue, -) -> SourceMetadataValue: - """Freeze one source-metadata value without erasing its nominal shape.""" - - if isinstance(value, Mapping): - return MappingProxyType( - { - str(key): source_metadata_scalar(nested_value) - for key, nested_value in value.items() - } - ) - return source_metadata_scalar(value) - - def _normalize_projection(projection: SourceProjection) -> None: """Normalize fields shared by every nominal source projection.""" @@ -1410,12 +1395,7 @@ def _normalize_projection(projection: SourceProjection) -> None: object.__setattr__( projection, "source_metadata", - MappingProxyType( - { - str(key): _normalized_source_metadata_value(value) - for key, value in projection.source_metadata.items() - } - ), + ResolvedSourceMetadataRecord.normalized_mapping(projection.source_metadata), ) object.__setattr__( projection, diff --git a/openhcs/core/source_workspace_projection.py b/openhcs/core/source_workspace_projection.py index 8bb9f20108..0b3216e025 100644 --- a/openhcs/core/source_workspace_projection.py +++ b/openhcs/core/source_workspace_projection.py @@ -24,7 +24,10 @@ NamedSourceBinding, SourceProjectionRole, ) -from openhcs.core.source_metadata import SourceMetadataMapping +from openhcs.core.source_metadata import ( + SourceMetadataFields, + SourceMetadataMapping, +) from openhcs.core.source_matching import ( source_component_metadata_values, source_metadata_value, @@ -553,12 +556,8 @@ def apply(self, payload: RuntimeArrayData) -> RuntimeArrayData: """Apply declared component metadata and source aliases to one payload.""" source_metadata = self.source_metadata if source_metadata is not None: - source_metadata = MappingProxyType( - { - field: value - for field, value in source_metadata.items() - if field != SOURCE_BINDING_ALIAS_METADATA_FIELD - } + source_metadata = SourceMetadataFields.with_fields( + source_metadata, {}, without=(SOURCE_BINDING_ALIAS_METADATA_FIELD,) ) current_metadata = image_payload_metadata(payload) metadata = self.metadata(current_metadata) @@ -839,7 +838,7 @@ def record_source_metadata( virtual_path: str, metadata_fields: SourceMetadataMapping, ) -> None: - normalized_metadata = MappingProxyType(dict(metadata_fields)) + normalized_metadata = SourceMetadataFields.readonly_snapshot(metadata_fields) self.source_metadata_by_path[virtual_path] = normalized_metadata self.source_metadata_by_path[str(self.plate_path / virtual_path)] = ( normalized_metadata diff --git a/openhcs/core/steps/abstract.py b/openhcs/core/steps/abstract.py index e82003ec93..19402fa4d3 100644 --- a/openhcs/core/steps/abstract.py +++ b/openhcs/core/steps/abstract.py @@ -8,10 +8,11 @@ import abc import inspect +from pathlib import Path from collections.abc import Mapping -from dataclasses import dataclass, is_dataclass +from dataclasses import dataclass, field, is_dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, get_type_hints +from typing import TYPE_CHECKING, Iterable, get_type_hints from objectstate import get_base_type_for_lazy, semantic_values_equal @@ -49,10 +50,29 @@ class StepExecutionObservation: tuple[RuntimeArtifactLocation, ...], ] + runtime_export_paths: tuple[Path, ...] = field(default_factory=tuple) + @classmethod def empty(cls) -> "StepExecutionObservation": return cls(MappingProxyType({})) + @classmethod + def combine( + cls, observations: Iterable["StepExecutionObservation"] + ) -> "StepExecutionObservation": + locations = {} + paths = [] + for observation in observations: + for ( + address, + values, + ) in observation.materialized_locations_by_address.items(): + locations[address] = tuple( + dict.fromkeys((*locations.get(address, ()), *values)) + ) + paths.extend(observation.runtime_export_paths) + return cls(MappingProxyType(locations), tuple(dict.fromkeys(paths))) + # def get_step_id(step: 'AbstractStep') -> str: # """ diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index 7d4a0da320..c82eab9015 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, cast import numpy as np @@ -50,6 +51,7 @@ with_source_component_metadata, ) from openhcs.core.source_projection import OpenHCSPlaneAddress +from openhcs.core.steps.abstract import StepExecutionObservation from openhcs.core.steps.function_output_identity import ( FunctionOutputIdentity, FunctionOutputIdentityAuthority, @@ -71,9 +73,11 @@ MaterializationSpec, MaterializationValue, Output, + SavedMaterializationOutputs, RawBackendKwargs, ViewerStreamBackendCallKwargs, materialization_outputs, + prepare_materialization, ) if TYPE_CHECKING: @@ -454,10 +458,10 @@ def metadata_items( fallback_source_identity or ArtifactStreamSourceMetadataAuthority.payload_source_identity(data) ) - emitted_identities = materialization_spec.emitted_source_identities(data) - if emitted_identities: + stream_identities = materialization_spec.stream_source_identities(data) + if stream_identities: return StreamSourceComponentMetadataItems.from_source_identities( - emitted_identities, + stream_identities, fallback_source_identity=fallback_source_identity, ) return StreamSourceComponentMetadataItems.from_values( @@ -754,7 +758,7 @@ def record_metadata_with_runtime_scope( ) elif scope.component is not None: component_metadata = with_source_component_metadata( - dict(metadata.source_component_metadata or {}), + metadata.source_component_metadata or {}, scope.component, scope.require_value_text(), ) @@ -817,6 +821,9 @@ def record_source_descriptor( variable_components=plan.variable_components, ) if identity is not None: + identity = materialization_spec.filename_identity_for_output( + identity, output_plan, + ) try: filename = Path( FunctionOutputPathAuthority.filename_for_identity( @@ -1221,18 +1228,6 @@ def runtime_artifact_materializations( return tuple(materializations) -def observed_runtime_artifact_materializations( - plan: CompiledStepPlan, - context: "ProcessingContext", -) -> tuple[RuntimeArtifactMaterialization, ...]: - """Derive historical materializations from the runtime observation ledger.""" - return runtime_artifact_materializations_from_records( - plan, - context, - context.runtime_value_store.observed_values, - ) - - def runtime_artifact_materializations_from_records( plan: CompiledStepPlan, context: "ProcessingContext", @@ -1271,62 +1266,12 @@ def runtime_artifact_materializations_from_records( return tuple(materializations) -def materialized_artifact_output_paths( - plan: CompiledStepPlan, - context: "ProcessingContext", -) -> tuple[Path, ...]: - """Re-derive exact persistent output paths from runtime materializations.""" - - if not plan.runtime_artifact_materialization.has_persistent_target: - return () - return tuple( - Path(output.path) - for materialization in runtime_artifact_materializations(plan, context) - if materialization.spec.participates_in_persistent_materialization() - for output in materialization.outputs(plan, context) - ) - - -def runtime_export_artifact_output_paths( - plan: CompiledStepPlan, - context: "ProcessingContext", -) -> tuple[Path, ...]: - """Derive exact pipeline-declared export paths from runtime materializations.""" - - if not plan.runtime_artifact_materialization.has_persistent_target: - return () - return tuple( - Path(output.path) - for materialization in runtime_artifact_materializations(plan, context) - if materialization.spec.participates_in_runtime_export_observation() - for output in materialization.outputs(plan, context) - ) - - -def observed_materialized_artifact_output_paths( - plan: CompiledStepPlan, - context: "ProcessingContext", - records: tuple[StoredRuntimeValue, ...], -) -> tuple[Path, ...]: - """Derive exact persistent outputs from one execution-owned observation.""" - - return tuple( - Path(location.path) - for locations in observed_materialized_artifact_locations_by_address( - plan, - context, - records, - ).values() - for location in locations - ) - - -def observed_runtime_export_artifact_output_paths( +def preview_reused_runtime_export_paths( plan: CompiledStepPlan, context: "ProcessingContext", records: tuple[StoredRuntimeValue, ...], ) -> tuple[Path, ...]: - """Derive exact pipeline-declared exports from worker-observed records.""" + """Predict historical debug-reuse exports, independently of new save outcomes.""" if not plan.runtime_artifact_materialization.has_persistent_target: return () @@ -1342,12 +1287,12 @@ def observed_runtime_export_artifact_output_paths( ) -def observed_materialized_artifact_locations_by_address( +def preview_reused_materialized_artifact_locations( plan: CompiledStepPlan, context: "ProcessingContext", records: tuple[StoredRuntimeValue, ...], ) -> Mapping[RuntimeArtifactAddress, tuple[RuntimeArtifactLocation, ...]]: - """Project exact persistent destinations for observed runtime artifacts.""" + """Predict historical debug-reuse destinations from retained logical values.""" if not plan.runtime_artifact_materialization.has_persistent_target: return {} @@ -1432,15 +1377,41 @@ def _planned_materialization_path( ) +@dataclass(frozen=True, slots=True) +class MaterializedRuntimeArtifact(SavedMaterializationOutputs): + """Actual writer outputs saved for one reduced runtime artifact.""" + + materialization: RuntimeArtifactMaterialization + + def observation(self, plan: CompiledStepPlan) -> StepExecutionObservation: + target = plan.runtime_artifact_materialization + if not target.has_persistent_target: + return StepExecutionObservation.empty() + backend = target.require_persistent_backend() + outputs = self.outputs_for_backend(backend) + if not outputs: + return StepExecutionObservation.empty() + address = RuntimeArtifactAddress.from_record(self.materialization.record) + locations = tuple( + RuntimeArtifactLocation(path=output.path, backend=backend) + for output in outputs + ) + paths = ( + tuple(Path(output.path) for output in outputs) + if self.materialization.spec.participates_in_runtime_export_observation() + else () + ) + return StepExecutionObservation(MappingProxyType({address: locations}), paths) + + def materialize_artifact_outputs( filemanager: "FileManager", plan: CompiledStepPlan, target_plan: ArtifactMaterializationTargetPlan, context: "ProcessingContext", -) -> None: - """Materialize planned artifact outputs to persistent and streaming backends.""" - from openhcs.processing.materialization import materialize - +) -> tuple[MaterializedRuntimeArtifact, ...]: + """Save each exact artifact batch once and return its successful outputs.""" + saved_materializations = [] images_dir = plan.artifact_images_dir for materialization in runtime_artifact_materializations(plan, context): @@ -1458,7 +1429,7 @@ def materialize_artifact_outputs( ) if not backends: continue - materialize( + batch = prepare_materialization( materialization.spec, data, str(materialization.base_path), @@ -1488,3 +1459,12 @@ def materialize_artifact_outputs( pipeline_position=plan.pipeline_position, output_plan=materialization.output_plan, ) + saved_materializations.append( + MaterializedRuntimeArtifact( + outputs_by_backend=MappingProxyType( + dict(batch.save().outputs_by_backend) + ), + materialization=materialization, + ) + ) + return tuple(saved_materializations) diff --git a/openhcs/core/steps/function_execution.py b/openhcs/core/steps/function_execution.py index 1b075b9d82..6cd8cf743c 100644 --- a/openhcs/core/steps/function_execution.py +++ b/openhcs/core/steps/function_execution.py @@ -7,7 +7,7 @@ import time import traceback from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from itertools import zip_longest from typing import TYPE_CHECKING @@ -28,6 +28,7 @@ ) from openhcs.core.progress import ProgressPhase, ProgressStatus, emit from openhcs.core.runtime_pattern_cache import RuntimePatternDiscoveryCacheKey +from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache from openhcs.core.source_binding_selection import ( SourceBoundAnchorPatternPolicy, SourceCandidatePath, @@ -223,6 +224,9 @@ class StepAnchorPatternFilter: output_manifest: StepOutputManifestStore source_workspace_authority: VirtualWorkspaceSourceProjectionAuthority source_workspace_projection_cache: VirtualWorkspaceSourceProjectionCache + source_binding_context_cache: RuntimeSourceBindingContextCache = field( + default_factory=RuntimeSourceBindingContextCache + ) @classmethod def from_context( @@ -243,6 +247,7 @@ def from_context( source_workspace_projection_cache=( context.runtime_source_workspace_projection_cache ), + source_binding_context_cache=context.runtime_source_binding_context_cache, ) def filtered(self, grouped_patterns: PatternGroups) -> PatternGroups: @@ -611,7 +616,7 @@ def source_pattern_context(self) -> SourcePatternResolutionContext: """Return source-path context used to filter source-bound anchors.""" projection = self.source_workspace_authority.projection_or_empty() - return SourcePatternResolutionContext.from_projection( + return self.source_binding_context_cache.source_pattern_context( parser=self.parser, projection=self.source_workspace_projection_cache.filtered_by_axis( projection, diff --git a/openhcs/core/steps/function_output_identity.py b/openhcs/core/steps/function_output_identity.py index f83da45f2f..a9775369ec 100644 --- a/openhcs/core/steps/function_output_identity.py +++ b/openhcs/core/steps/function_output_identity.py @@ -23,9 +23,9 @@ SourceImageIdentity, SourceImageProvenanceIdentity, ) +from openhcs.core.source_metadata import SourceMetadataFields from openhcs.core.source_matching import ( source_component_metadata_items, - with_source_component_metadata, ) from openhcs.core.source_projection import OpenHCSPlaneAddress from openhcs.microscopes.microscope_interfaces import FilenameParser @@ -144,13 +144,15 @@ def component_metadata( source_metadata: SourceComponentMetadata | None = None, ) -> SourceComponentMetadata: """Return parser-compatible component metadata for this output identity.""" - metadata = dict(source_metadata or {}) - metadata.update(self.component_values) - for component, value in source_component_metadata_items(self.component_values): - metadata = with_source_component_metadata(metadata, component, value) - if self.extension is not None: - metadata["extension"] = self.extension - return metadata + metadata = source_metadata if source_metadata is not None else {} + return SourceMetadataFields.with_fields( + SourceMetadataFields.composition_snapshot(metadata), + self.component_values, + components=source_component_metadata_items(self.component_values), + after_components=( + {"extension": self.extension} if self.extension is not None else None + ), + ) def filename_component_metadata(self) -> SourceComponentMetadata: """Return parser-compatible metadata used to construct the output filename.""" diff --git a/openhcs/core/steps/function_output_manifest.py b/openhcs/core/steps/function_output_manifest.py index ea93bbff1e..daef36dea2 100644 --- a/openhcs/core/steps/function_output_manifest.py +++ b/openhcs/core/steps/function_output_manifest.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import ClassVar, Iterator, Sequence from weakref import WeakKeyDictionary @@ -23,6 +23,7 @@ ImagePayloadMetadata, image_payload_metadata, ) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.step_dependencies import StepInputDependencyKind from openhcs.core.steps.function_output_identity import ( @@ -113,6 +114,19 @@ class ProducedOutputSemantics(FunctionOutputIdentity): output_path: str relative_output_path: str image_metadata: ImagePayloadMetadata | None = None + main_flow_plane_axis: RuntimePlaneAxis | None = RuntimePlaneAxis.RUNTIME_SLICE + + def passed_through(self, plan: CompiledStepPlan) -> "ProducedOutputSemantics": + """Retain the image domain and physical identity under the next producer.""" + return replace( + self, + producer_identity=FunctionStepOutputProducerIdentityAuthority.build( + FunctionStepOutputProducerIdentityRequest.from_main_flow( + plan, self.output_context, + ) + ), + relative_output_path=source_path_identity(self.output_path).name, + ) def contextualize_image_payload( self, @@ -130,6 +144,26 @@ def path_under(self, output_dir: str | Path) -> str: return str(Path(output_dir) / self.relative_output_path) + def owns_persisted_artifact( + self, + output_plan: ArtifactOutputPlan, + output_path: str, + output_dir: str | Path, + ) -> bool: + """Identify the same declared producer and exact saved occurrence.""" + identity = self.producer_identity + return ( + self.output_context.persisted_source_alias, + identity.artifact_kind, + identity.step_scope_id, + Path(self.path_under(output_dir)), + ) == ( + output_plan.name, + output_plan.artifact_type.value, + output_plan.producer_step_scope_id, + Path(output_path), + ) + @property def output_context(self) -> AlignedImageSliceContext: """Return the declared main-flow context for this produced output.""" @@ -175,6 +209,7 @@ def from_output( output_identity: FunctionOutputIdentity, output_context: AlignedImageSliceContext | None = None, image_metadata: ImagePayloadMetadata | None = None, + main_flow_plane_axis: RuntimePlaneAxis | None = RuntimePlaneAxis.RUNTIME_SLICE, ) -> "ProducedOutputSemantics": output_path_text = str(output_path) if output_context is None: @@ -197,6 +232,7 @@ def from_output( Path(plan.output_dir), ), image_metadata=image_metadata, + main_flow_plane_axis=main_flow_plane_axis, ) @classmethod @@ -407,14 +443,27 @@ def producer_output_contexts_for_paths( ) -> tuple[AlignedImageSliceContext, ...]: """Return producer output contexts aligned to concrete input paths.""" - producer_records = self._selected_unique_producer_records_for(plan) - if producer_records is None: + records = self.producer_output_records_for_paths(plan, paths, parser) + if records is None: return tuple( AlignedImageSliceContext.anonymous_main_flow() for _path in paths ) + return tuple(record.output_context for record in records) + + def producer_output_records_for_paths( + self, + plan: CompiledStepPlan, + paths: Sequence[str], + parser: FilenameParser, + ) -> tuple[ProducedOutputSemantics, ...] | None: + """Resolve exact producer declarations in the requested physical path order.""" + + producer_records = self._selected_unique_producer_records_for(plan) + if producer_records is None: + return None path_index = ProducedPathRecordIndex.from_records(producer_records, parser) - contexts: list[AlignedImageSliceContext] = [] + records: list[ProducedOutputSemantics] = [] for path in paths: matching_records = path_index.matching_records(path) if len(matching_records) != 1: @@ -422,8 +471,8 @@ def producer_output_contexts_for_paths( "Expected one producer output context for input path " f"{path!r}, found {len(matching_records)}." ) - contexts.append(matching_records[0].output_context) - return tuple(contexts) + records.append(matching_records[0]) + return tuple(records) def filter_to_producer_paths( self, @@ -501,6 +550,12 @@ def _unique_output_path_records( ) -> tuple[ProducedOutputSemantics, ...]: records_by_path: dict[str, ProducedOutputSemantics] = {} for record in records: + existing = records_by_path.get(record.output_path) + if existing is not None and existing.main_flow_plane_axis is not record.main_flow_plane_axis: + raise ValueError( + "One produced output path cannot declare conflicting main-flow image axes: " + f"{record.output_path!r}." + ) records_by_path.setdefault(record.output_path, record) return tuple(records_by_path.values()) diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py index 919d4acaa5..fcaef7bf7f 100644 --- a/openhcs/core/steps/function_outputs.py +++ b/openhcs/core/steps/function_outputs.py @@ -6,9 +6,10 @@ import time from abc import ABC, abstractmethod from collections.abc import Callable, Mapping -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from pathlib import Path -from typing import ClassVar +from types import MappingProxyType +from typing import ClassVar, TypeVar import numpy as np from metaclass_registry import ( @@ -52,14 +53,16 @@ OpenHCSPlaneAddress, SourceArtifactProjection, SourcePlaneProjection, + SourceProjection, SourceProjectionMetadataSerializer, SourceProjectionSet, ) +from openhcs.core.steps.abstract import StepExecutionObservation from openhcs.core.steps.function_artifact_materialization import ( + MaterializedRuntimeArtifact, PersistentArtifactMaterializationTargetPlan, StreamingOnlyArtifactMaterializationTargetPlan, materialize_artifact_outputs, - runtime_artifact_materializations, ) from openhcs.core.steps.function_io import ( prepare_storage_image_payloads, @@ -137,50 +140,61 @@ def memory_path( def finalize_function_step_outputs( context: ProcessingContext, plan: CompiledStepPlan, -) -> None: - """Persist images, streams, metadata, and non-image artifacts for one step.""" +) -> StepExecutionObservation: + """Save and publish one step's actual outputs before releasing their payloads.""" if not RuntimeProfileLogger.enabled(): MemoryOutputWriter.write_if_needed(context, plan) MaterializedImageOutputWriter.write_if_needed(context, plan) StreamOutputsAuthority.stream_outputs(context, plan) - RuntimeArtifactMaterializationAuthority.materialize(context, plan) - OpenHCSMetadataWriter.write(context, plan) - return - - _profile_finalization_phase( - "finalize_memory_outputs", - lambda: MemoryOutputWriter.write_if_needed(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_materialized_images", - lambda: MaterializedImageOutputWriter.write_if_needed(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_stream_outputs", - lambda: StreamOutputsAuthority.stream_outputs(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_runtime_artifacts", - lambda: RuntimeArtifactMaterializationAuthority.materialize(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_openhcs_metadata", - lambda: OpenHCSMetadataWriter.write(context, plan), - plan, + materializations = RuntimeArtifactMaterializationAuthority.materialize( + context, plan + ) + OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ) + else: + _profile_finalization_phase( + "finalize_memory_outputs", + lambda: MemoryOutputWriter.write_if_needed(context, plan), + plan, + ) + _profile_finalization_phase( + "finalize_materialized_images", + lambda: MaterializedImageOutputWriter.write_if_needed(context, plan), + plan, + ) + _profile_finalization_phase( + "finalize_stream_outputs", + lambda: StreamOutputsAuthority.stream_outputs(context, plan), + plan, + ) + materializations = _profile_finalization_phase( + "finalize_runtime_artifacts", + lambda: RuntimeArtifactMaterializationAuthority.materialize(context, plan), + plan, + ) + _profile_finalization_phase( + "finalize_openhcs_metadata", + lambda: OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ), + plan, + ) + return StepExecutionObservation.combine( + item.observation(plan) for item in materializations ) +_FinalizationResult = TypeVar("_FinalizationResult") + + def _profile_finalization_phase( label: str, - operation: Callable[[], None], + operation: Callable[[], _FinalizationResult], plan: CompiledStepPlan, -) -> None: +) -> _FinalizationResult: started_at = time.perf_counter() - operation() + result = operation() RuntimeProfileLogger.log( logger, label, @@ -189,6 +203,7 @@ def _profile_finalization_phase( step_name=plan.step_name, axis_id=plan.axis_id, ) + return result class MemoryOutputWriter: @@ -691,6 +706,9 @@ class OutputTarget(ABC, metaclass=AutoRegisterMeta): plate_root: str sub_dir: str results_dir: str | None + artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = field( + default=(), compare=False, hash=False, repr=False + ) @classmethod @abstractmethod @@ -719,17 +737,25 @@ def from_execution( @classmethod def for_execution( - cls, context: ProcessingContext, plan: CompiledStepPlan + cls, + context: ProcessingContext, + plan: CompiledStepPlan, + *, + artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = (), ) -> tuple[OpenHCSMetadataWriter.OutputTarget, ...]: return tuple( target for declaration in cls.__registry__.values() if (owner := declaration.from_execution(context, plan)) is not None - for target in owner.production_targets(context, plan) + for target in replace( + owner, artifact_materializations=artifact_materializations + ).production_targets(context, plan) ) def production_targets( - self, context: ProcessingContext, plan: CompiledStepPlan + self, + context: ProcessingContext, + plan: CompiledStepPlan, ) -> tuple[OpenHCSMetadataWriter.OutputTarget, ...]: """Project this declaration's exact storage destinations for the step.""" return (self,) @@ -747,27 +773,36 @@ def produced_records( return () def runtime_artifact_projection_paths( - self, context: ProcessingContext, plan: CompiledStepPlan + self, + context: ProcessingContext, + plan: CompiledStepPlan, + *, + produced_projections: Mapping[ + Path, tuple[ProducedOutputSemantics, SourceProjection] + ] = MappingProxyType({}), ) -> tuple[tuple[SourceArtifactProjection, str], ...]: """Publish artifacts persisted in this declared storage target.""" materialization = plan.runtime_artifact_materialization if materialization.persists_to_backend(self.backend): - return self.project_runtime_artifacts(context, plan) + return self.project_runtime_artifacts( + context, plan, produced_projections=produced_projections + ) return () - def contains_images(self, context: ProcessingContext) -> bool: - """Return whether the completed target contains image outputs.""" + def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: + """Return this declaration's outputs eligible for publication.""" + return tuple( + context.filemanager.list_image_files(self.output_dir, self.backend) + ) + + def contains_outputs(self, context: ProcessingContext) -> bool: + """Return whether this declared destination has publishable outputs.""" if context.filemanager is None: raise ValueError("OpenHCS metadata requires a file manager.") if not context.filemanager.is_dir(self.output_dir, self.backend): return False - return bool( - context.filemanager.list_image_files( - self.output_dir, - self.backend, - ) - ) + return bool(self.stored_output_paths(context)) def write( self, @@ -806,7 +841,10 @@ def write( component_labels=context.metadata_cache, backend=self.backend, is_main=self.is_main, - results_dir=(Path(self.results_dir).name if self.results_dir else None), + results_dir=( + str(Path(self.results_dir).relative_to(self.plate_root)) + if self.results_dir is not None else None + ), ) def produced_projection_metadata( @@ -820,7 +858,9 @@ def produced_projection_metadata( if context.filemanager is None: raise ValueError("OpenHCS metadata requires a file manager.") target = type(self).from_plan(plan) - if target is None or self not in target.production_targets(context, plan): + if target is None or self not in replace( + target, artifact_materializations=self.artifact_materializations + ).production_targets(context, plan): raise ValueError( "Produced metadata plan does not own this output target." ) @@ -837,6 +877,7 @@ def produced_projection_metadata( else () ) projection_paths = [] + produced_projections = {} declared_addresses: set[OpenHCSPlaneAddress] = set() parser_context = FunctionOutputParserContext.from_processing_context( context @@ -863,12 +904,16 @@ def produced_projection_metadata( SourcePlaneProjection( address=address, ref=SourcePixelRef(self.backend, virtual_path), + source_alias=record.output_context.persisted_source_alias, source_metadata=source_metadata, image_metadata=metadata, ), virtual_path, ) ) + produced_projections[Path(destination)] = ( + record, projection_paths[-1][0] + ) continue projection_paths.append( ( @@ -883,8 +928,13 @@ def produced_projection_metadata( virtual_path, ) ) + produced_projections[Path(destination)] = ( + record, projection_paths[-1][0] + ) projection_paths.extend( - self.runtime_artifact_projection_paths(context, plan) + self.runtime_artifact_projection_paths( + context, plan, produced_projections=produced_projections + ) ) if not projection_paths: return None @@ -906,20 +956,17 @@ def project_runtime_artifacts( self, context: ProcessingContext, plan: CompiledStepPlan, + *, + produced_projections: Mapping[ + Path, tuple[ProducedOutputSemantics, SourceProjection] + ] = MappingProxyType({}), ) -> tuple[tuple[SourceArtifactProjection, str], ...]: """Project persisted image artifacts into the target source authority.""" projection_paths = [] - for materialization in runtime_artifact_materializations(plan, context): - if ( - not materialization.spec.participates_in_persistent_materialization() - ): - continue - for output in materialization.outputs( - plan, - context, - output_path_filter=ImageFileFormat.is_image_path, - ): + for saved_artifact in self.artifact_materializations: + materialization = saved_artifact.materialization + for output in saved_artifact.outputs_for_backend(self.backend): if not ImageFileFormat.is_image_path(output.path) or Path( output.path ).parent != Path(self.output_dir): @@ -942,6 +989,21 @@ def project_runtime_artifacts( metadata ) ) + produced = produced_projections.get(Path(destination)) + if produced is not None: + record, projection = produced + if record.owns_persisted_artifact( + materialization.output_plan, destination, self.output_dir + ): + if ( + projection.address != address + or projection.image_metadata != metadata + ): + raise ValueError( + "Conflicting metadata for persisted image " + f"occurrence {destination!r}." + ) + continue source_metadata = metadata.source_component_metadata or {} persisted_source_metadata = dict(source_metadata) metadata.source_voxel_spacing.merge_into( @@ -1005,8 +1067,12 @@ def write( cls, context: ProcessingContext, plan: CompiledStepPlan, + *, + artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = (), ) -> None: - for target in cls.OutputTarget.for_execution(context, plan): + for target in cls.OutputTarget.for_execution( + context, plan, artifact_materializations=artifact_materializations + ): if not plan.create_openhcs_metadata: structured_metadata = target.produced_projection_metadata(context, plan) if structured_metadata is None: @@ -1038,7 +1104,7 @@ def finalize_completed_plate( for owner, context in target_contexts.items(): for target in owner.reconciliation_targets(context): - if target.contains_images(context): + if target.contains_outputs(context): target.write(context) @@ -1121,32 +1187,36 @@ def from_plan( class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): - """Image artifacts persist independently of main-flow image/checkpoint storage.""" + """Saved artifacts own result destinations independently of image storage.""" + + def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: + """Include every saved format in this declared result destination.""" + return tuple(context.filemanager.list_files(self.output_dir, self.backend)) def production_targets( - self, context: ProcessingContext, plan: CompiledStepPlan + self, + context: ProcessingContext, + plan: CompiledStepPlan, ) -> tuple[RuntimeArtifactMetadataTarget, ...]: """Derive directories from the same writer outputs used to save artifacts.""" directories = dict.fromkeys( Path(output.path).parent - for materialization in runtime_artifact_materializations(plan, context) - if materialization.spec.participates_in_persistent_materialization() - for output in materialization.outputs( - plan, context, output_path_filter=ImageFileFormat.is_image_path - ) - if ImageFileFormat.is_image_path(output.path) + for materialization in self.artifact_materializations + for output in materialization.outputs_for_backend(self.backend) ) return tuple( target for directory in directories for target in (self.for_directory(directory),) - if target.contains_images(context) + if target.contains_outputs(context) ) def reconciliation_targets( self, context: ProcessingContext ) -> tuple[RuntimeArtifactMetadataTarget, ...]: """Use durable typed projections, without reloading cleaned artifact values.""" + from openhcs.microscopes.openhcs import OpenHCSMetadataHandler + subdirectories = OpenHCSMetadataSubdirectories.from_path( METADATA_CONFIG.metadata_path(self.plate_root) ) @@ -1159,15 +1229,26 @@ def reconciliation_targets( if isinstance(projection, SourceArtifactProjection) and projection.ref.backend == self.backend ) + result_directories = OpenHCSMetadataHandler( + context.filemanager + ).analysis_result_directories(self.plate_root) return tuple( self.for_directory(directory) - for directory in dict.fromkeys((self.output_dir, *directories)) + for directory in dict.fromkeys( + ( + self.output_dir, + *directories, + *(directory.path for directory in result_directories), + ) + ) ) def for_directory(self, directory: Path) -> RuntimeArtifactMetadataTarget: """Retain the compiled plate/backend while projecting one declared directory.""" sub_dir = directory.relative_to(self.plate_root) - return replace(self, output_dir=directory, sub_dir=str(sub_dir)) + return replace( + self, output_dir=directory, sub_dir=str(sub_dir), results_dir=str(directory) + ) @classmethod def from_plan(cls, plan: CompiledStepPlan) -> RuntimeArtifactMetadataTarget | None: @@ -1181,7 +1262,7 @@ def from_plan(cls, plan: CompiledStepPlan) -> RuntimeArtifactMetadataTarget | No backend=materialization.require_persistent_backend(), plate_root=plate_root, sub_dir=str(output_dir.relative_to(plate_root)), - results_dir=None, + results_dir=str(output_dir), ) @@ -1193,27 +1274,28 @@ def materialize( cls, context: ProcessingContext, plan: CompiledStepPlan, - ) -> None: + ) -> tuple[MaterializedRuntimeArtifact, ...]: if not plan.artifact_outputs: - return + return () materialization_plan = plan.runtime_artifact_materialization has_persistent_target = materialization_plan.has_persistent_target has_streaming_target = bool(plan.streaming_configs) if not has_persistent_target and not has_streaming_target: logger.info("Skipping runtime artifact materialization and streaming") - return + return () logger.info( "Starting materialization for %s artifact outputs", len(plan.artifact_outputs), ) - materialize_artifact_outputs( + materializations = materialize_artifact_outputs( context.filemanager, plan, cls.target_plan(materialization_plan), context, ) logger.info("Completed artifact materialization") + return materializations @staticmethod def target_plan( diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index c6ff44e790..e0fbc70fea 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -43,7 +43,6 @@ ) from openhcs.core.callable_contract import ( CallableRuntimeCacheKey, - prepare_processing_callable, ) from openhcs.core.component_group_scope import ( ComponentGroupScope, @@ -75,6 +74,7 @@ AlignedImageStack, AlignedImageSliceContext, ImagePayloadBundleContext, + ImagePayloadStackComposition, ImageOutputBundle, flatten_aligned_image_payload_slices, stack_image_payload_context, @@ -245,9 +245,6 @@ class FunctionInvocationCallableResolver: def prepare(cls, invocation: CompiledFunctionInvocation) -> None: """Resolve and cache one invocation callable before timed execution.""" cls.resolve(invocation) - prepare_processing_callable( - invocation.contract.resolve_canonical_raw_callable() - ) @classmethod def resolve(cls, invocation: CompiledFunctionInvocation) -> Callable: @@ -1580,6 +1577,69 @@ class PatternGroupData: ) + @staticmethod + def validate_main_flow_cohort( + producer_records: Sequence[ProducedOutputSemantics] | None, + ) -> None: + """One input cohort has one declared whole-image composition domain.""" + if producer_records and len({record.main_flow_plane_axis for record in producer_records}) != 1: + raise ValueError("One main-flow cohort cannot combine different declared image axes.") + + @classmethod + def from_loaded_images( + cls, + matching_files: list[str], + payloads: Sequence[RuntimeArrayData], + *, + producer_records: Sequence[ProducedOutputSemantics] | None, + source_binding_context: SourceBindingRuntimeContext, + execution_plan: CompiledStepPlan, + source_projection: VirtualWorkspaceSourceProjection | None, + workspace_source_lookups: Sequence[VirtualWorkspacePathLookup], + ) -> "PatternGroupData": + """Compose a selected admissible input cohort in its declared image domain.""" + if ( + producer_records + and len(producer_records) == 1 + and producer_records[0].main_flow_plane_axis is image_payload_metadata(payloads[0]).plane_axis + ): + main_data_stack = ImagePayloadStackComposition.copy_whole_image( + payloads[0], memory_type=execution_plan.input_memory_type, + device_id=execution_plan.device_id_for(execution_plan.input_memory_type), + ) + else: + metadata_mode = ImagePayloadMetadataCompositionMode.STACK + if producer_records: + declared_axis = producer_records[0].main_flow_plane_axis + metadata_mode = ( + ImagePayloadMetadataCompositionMode.BUNDLE + if declared_axis is None + else ImagePayloadMetadataCompositionMode.for_plane_axis(declared_axis) + ) + if source_projection is not None and workspace_source_lookups: + metadata_mode = source_projection.payload_composition_mode( + workspace_source_lookups + ) + if metadata_mode is ImagePayloadMetadataCompositionMode.STACK: + main_data_stack = stack_runtime_slices( + tuple(image_payload_data(payload) for payload in payloads), + execution_plan.input_memory_type, + execution_plan.device_id_for(execution_plan.input_memory_type), + ) + main_data_stack = stack_image_payload_context( + payloads, main_data_stack, metadata_mode=metadata_mode, + ) + elif metadata_mode is ImagePayloadMetadataCompositionMode.BUNDLE: + main_data_stack = ImagePayloadBundleContext.from_payloads( + tuple(payloads), metadata_mode=metadata_mode, + ).compose() + return cls( + matching_files=matching_files, + main_data_stack=main_data_stack, + source_binding_context=source_binding_context, + ) + + @dataclass(frozen=True, slots=True) class OutputPathBatchEntry: """Resolved identity for one output path in a runtime save batch.""" @@ -2084,7 +2144,7 @@ def execution_group_source_payload( if component is None or self.group_key is None: return source_payload metadata = image_payload_metadata(source_payload) - component_metadata = dict(metadata.source_component_metadata or {}) + component_metadata = metadata.source_component_metadata or {} component_metadata = with_source_component_metadata( component_metadata, component, @@ -2425,6 +2485,7 @@ class PatternGroupOutputData: slices: list[RuntimeArrayData] slice_contexts: tuple[AlignedImageSliceContext, ...] = () stack_payload: RuntimeArrayData | None = None + main_flow_source: RuntimeArrayData | None = None def __post_init__(self) -> None: if not self.slice_contexts: @@ -2437,6 +2498,69 @@ def __post_init__(self) -> None: f"got {len(self.slice_contexts)} context(s) for {len(self.slices)} slice(s)." ) + @classmethod + def from_single_image( + cls, + value: RuntimeArrayData, + *, + slice_contexts: tuple[AlignedImageSliceContext, ...] = (), + memory_type: MemoryType, + device_id: int | None, + ) -> "PatternGroupOutputData": + """Copy one whole image for main-flow reuse without declaring a new axis.""" + return cls( + slices=(value,), + slice_contexts=slice_contexts, + stack_payload=ImagePayloadStackComposition.copy_whole_image( + value, memory_type=memory_type, device_id=device_id, + ), + main_flow_source=value, + ) + + def plane_axis_for_output_context( + self, context: AlignedImageSliceContext, + ) -> RuntimePlaneAxis | None: + """Derive each persisted member's domain from its original nominal producer.""" + if isinstance(self.main_flow_source, AlignedImageStack): + return self.main_flow_source.plane_axis_for_output_context(context) + if isinstance(self.main_flow_source, ImagePayloadMetadataCarrier): + return image_payload_metadata(self.main_flow_source).plane_axis + return RuntimePlaneAxis.RUNTIME_SLICE + + def cache_payload_for_outputs( + self, + payloads: Sequence[RuntimeArrayData], + metadata: Sequence[ImagePayloadMetadata], + ) -> RuntimeArrayData | None: + """Keep the original whole-image domain while applying saved output context.""" + if self.stack_payload is None: + return None + data = image_payload_data(self.stack_payload) + if ( + len(payloads) == 1 + and self.plane_axis_for_output_context(self.slice_contexts[0]) is metadata[0].plane_axis + and np.shape(data) == np.shape(image_payload_data(payloads[0])) + ): + return metadata[0].replace_fields().payload_with( + data, image_payload_mask(self.stack_payload), + ) + if np.shape(data)[:1] != (len(payloads),): + raise ValueError( + "PatternGroupOutputData.stack_payload must match its declared " + f"output slice count: stack shape {np.shape(data)!r}, " + f"slice count {len(payloads)}." + ) + return stack_image_payload_context_from_metadata( + payloads, data, metadata, + metadata_mode=( + ImagePayloadMetadataCompositionMode.for_plane_axis( + image_payload_metadata(self.stack_payload).plane_axis, + ) + if isinstance(self.stack_payload, ImagePayloadMetadataCarrier) + else ImagePayloadMetadataCompositionMode.STACK + ), + ) + @classmethod def from_aligned_stack( cls, @@ -2448,15 +2572,32 @@ def from_aligned_stack( """Materialize correlated output slices through the alignment owner.""" projected_outputs = tuple(value.projected_output_slices()) payloads = [payload for payload, _context in projected_outputs] + metadata_mode = value.projected_output_composition_mode + if metadata_mode is None: + result = cls.from_single_image( + payloads[0], + slice_contexts=tuple(context for _payload, context in projected_outputs), + memory_type=memory_type, + device_id=device_id, + ) + result.main_flow_source = value + return result data = tuple(image_payload_data(payload) for payload in payloads) stack_payload = None - if len({tuple(np.shape(item)) for item in data}) == 1: - stacked = stack_runtime_slices(data, memory_type, device_id) - stack_payload = stack_image_payload_context( - payloads, - stacked, - metadata_mode=ImagePayloadMetadataCompositionMode.STACK, - ) + declared_axes = { + value.plane_axis_for_output_context(context) + for _payload, context in projected_outputs + } + if len(declared_axes) == 1 and len({tuple(np.shape(item)) for item in data}) == 1: + if metadata_mode is ImagePayloadMetadataCompositionMode.BUNDLE: + stack_payload = ImagePayloadBundleContext.from_payloads( + tuple(payloads), metadata_mode=metadata_mode, + ).compose() + else: + stacked = stack_runtime_slices(data, memory_type, device_id) + stack_payload = stack_image_payload_context( + payloads, stacked, metadata_mode=metadata_mode, + ) return cls( slices=payloads, slice_contexts=( @@ -2465,6 +2606,7 @@ def from_aligned_stack( else () ), stack_payload=stack_payload, + main_flow_source=value, ) def __iter__(self) -> Iterator[RuntimeArrayData]: @@ -2647,21 +2789,20 @@ def _record_main_flow_passthrough(self, matching_files: Sequence[str]) -> None: return plan = self.request.execution_plan parser = self.request.context.microscope_handler.parser - output_contexts = self._producer_output_contexts(matching_files) - records = tuple( - ProducedOutputSemantics.from_existing_main_flow_path( - plan, - self._input_memory_path(plan.input_dir, matching_file), - parser, - output_context=output_context, - ) - for matching_file, output_context in zip( - matching_files, - output_contexts, - strict=True, + manifest = step_output_manifest(self.request.context) + producer_records = manifest.producer_output_records_for_paths(plan, matching_files, parser) + if producer_records is None: + records = tuple( + ProducedOutputSemantics.from_existing_main_flow_path( + plan, + self._input_memory_path(plan.input_dir, matching_file), + parser, + ) + for matching_file in matching_files ) - ) - step_output_manifest(self.request.context).record_outputs(plan, records) + else: + records = tuple(record.passed_through(plan) for record in producer_records) + manifest.record_outputs(plan, records) def _producer_output_contexts( self, @@ -2761,6 +2902,14 @@ def _load_input_stack(self) -> PatternGroupData: matching_files=matching_files, source_projection=source_projection, ).runtime_context() + producer_records = ( + output_manifest.producer_output_records_for_paths( + plan, matching_files, context.microscope_handler.parser, + ) + if plan.main_input_dependency.kind is StepInputDependencyKind.STEP_OUTPUT + else None + ) + PatternGroupData.validate_main_flow_cohort(producer_records) cached_stack = context.runtime_image_stack_cache.get( tuple(full_file_paths), memory_type=plan.input_memory_type, @@ -2797,32 +2946,17 @@ def _load_input_stack(self) -> PatternGroupData: f"Check file integrity and format compatibility." ) - metadata_mode = ImagePayloadMetadataCompositionMode.STACK - if source_projection is not None and workspace_source_lookups: - metadata_mode = source_projection.payload_composition_mode( - workspace_source_lookups - ) - if metadata_mode is ImagePayloadMetadataCompositionMode.STACK: - raw_slice_data = tuple( - image_payload_data(slice_data) for slice_data in raw_slices - ) - main_data_stack = stack_runtime_slices( - raw_slice_data, - plan.input_memory_type, - plan.device_id_for(plan.input_memory_type), - ) - main_data_stack = stack_image_payload_context( - raw_slices, - main_data_stack, - metadata_mode=metadata_mode, - ) - elif metadata_mode is ImagePayloadMetadataCompositionMode.BUNDLE: - main_data_stack = ImagePayloadBundleContext.from_payloads( - tuple(raw_slices), - metadata_mode=metadata_mode, - ).compose() + return PatternGroupData.from_loaded_images( + matching_files, + raw_slices, + producer_records=producer_records, + source_binding_context=source_binding_context, + execution_plan=plan, + source_projection=source_projection, + workspace_source_lookups=workspace_source_lookups, + ) else: - main_data_stack = cached_stack.stack + main_data_stack = cached_stack return PatternGroupData( matching_files=matching_files, @@ -2949,7 +3083,7 @@ def _source_binding_load_universe(self) -> tuple[str, ...]: def _source_binding_candidate_context(self) -> SourcePatternResolutionContext: projection = self.source_workspace_projection_authority().projection_or_empty() - return SourcePatternResolutionContext.from_projection( + return self.request.context.runtime_source_binding_context_cache.source_pattern_context( parser=self.request.context.microscope_handler.parser, projection=self.source_workspace_projection_cache().filtered_by_axis( projection, @@ -3121,16 +3255,8 @@ def _validate_and_unstack( and processed_stack.metadata.plane_axis is None ): output_context = self._unwrapped_main_flow_output_context() - scalar_data = image_payload_data(processed_stack) - stacked_data = stack_runtime_slices( - (scalar_data,), - self.request.execution_plan.output_memory_type, - self.request.execution_plan.device_id_for( - self.request.execution_plan.output_memory_type - ), - ) - return PatternGroupOutputData( - slices=(processed_stack,), + return PatternGroupOutputData.from_single_image( + processed_stack, slice_contexts=( (output_context,) if output_context is not None @@ -3140,10 +3266,9 @@ def _validate_and_unstack( else () ) ), - stack_payload=stack_image_payload_context( - (processed_stack,), - stacked_data, - metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + memory_type=self.request.execution_plan.output_memory_type, + device_id=self.request.execution_plan.device_id_for( + self.request.execution_plan.output_memory_type ), ) if isinstance(processed_stack, AlignedImageStack): @@ -3233,6 +3358,7 @@ def _validate_and_unstack( slices=output_payloads, slice_contexts=slice_contexts, stack_payload=processed_stack, + main_flow_source=processed_stack, ) def _unwrapped_main_flow_output_context( @@ -3340,6 +3466,7 @@ def _save_outputs( output_identity, output_context=output_context, image_metadata=output_metadata, + main_flow_plane_axis=output_data.plane_axis_for_output_context(output_context), ) if output_directory_exists and context.filemanager.exists( @@ -3377,24 +3504,10 @@ def _save_outputs( output_paths_batch, Backend.MEMORY.value, ) - stack_payload_data = ( - image_payload_data(output_data.stack_payload) - if output_data.stack_payload is not None - else None + stack_payload = output_data.cache_payload_for_outputs( + output_payloads, output_payload_metadata, ) - if output_data.stack_payload is not None: - if np.shape(stack_payload_data)[:1] != (len(output_payloads),): - raise ValueError( - "PatternGroupOutputData.stack_payload must match its declared " - f"output slice count: stack shape {np.shape(stack_payload_data)!r}, " - f"slice count {len(output_payloads)}." - ) - stack_payload = stack_image_payload_context_from_metadata( - output_payloads, - stack_payload_data, - output_payload_metadata, - metadata_mode=ImagePayloadMetadataCompositionMode.STACK, - ) + if stack_payload is not None: context.runtime_image_stack_cache.store( tuple(output_paths_batch), memory_type=self.request.execution_plan.output_memory_type, diff --git a/openhcs/core/viewer_streaming_service.py b/openhcs/core/viewer_streaming_service.py index d878d3146c..89f80d44e2 100644 --- a/openhcs/core/viewer_streaming_service.py +++ b/openhcs/core/viewer_streaming_service.py @@ -846,7 +846,7 @@ def stream_rois( point_domain = ROIFractionalZ.source_component_domain(rois, metadata) if point_domain is not None: point_domains[filename] = point_domain - data_list.append(ROIArchiveSourceMetadata.geometry(rois)) + data_list.append(rois) paths.append(filename) loaded_indices.append(i - 1) diff --git a/openhcs/core/virtual_workspace_metadata.py b/openhcs/core/virtual_workspace_metadata.py index 6ef9709f73..fc0c25cee3 100644 --- a/openhcs/core/virtual_workspace_metadata.py +++ b/openhcs/core/virtual_workspace_metadata.py @@ -5,7 +5,7 @@ import json import logging import os -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType @@ -21,9 +21,9 @@ from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.source_bindings import SourceProjectionRole from openhcs.core.source_metadata import ( + DurableSourceMetadata, SourceMetadataMapping, SourceMetadataScalar, - SourceMetadataValue, SourceVoxelSpacing, ) from openhcs.core.source_projection import ( @@ -97,7 +97,12 @@ def update(data: dict[str, Any] | None) -> dict[str, Any]: } else: subdirectory[key] = value - self._update_projection_geometry(subdirectory) + self._update_projection_geometry( + subdirectory, + VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries.values(), + ) return data self._execute_update( @@ -139,7 +144,12 @@ def update(data): subdirectory_name, {} ) self._merge_source_projection_fields(subdirectory, projection_metadata) - self._update_projection_geometry(subdirectory) + self._update_projection_geometry( + subdirectory, + VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries.values(), + ) return data self._execute_update(metadata_path, update) @@ -222,37 +232,45 @@ def update(data): ) ) subdirectory.update(serializer.projection_fields(retained_paths)) - projections = SourceProjectionSet( - tuple(entries[path] for path in published_paths) - ) - subdirectory.update( - serializer.component_metadata(projections, labels=component_labels) - ) + if published_paths: + projections = SourceProjectionSet( + tuple(entries[path] for path in published_paths) + ) + subdirectory.update( + serializer.component_metadata(projections, labels=component_labels) + ) subdirectory[FIELDS.IMAGE_FILES] = list(published_paths) subdirectory[FIELDS.MICROSCOPE_HANDLER_NAME] = microscope_handler_name - subdirectory[FIELDS.SOURCE_FILENAME_PARSER_NAME] = source_filename_parser_name + subdirectory[FIELDS.SOURCE_FILENAME_PARSER_NAME] = ( + source_filename_parser_name + ) subdirectory[FIELDS.AVAILABLE_BACKENDS] = { - **subdirectory.get(FIELDS.AVAILABLE_BACKENDS, {}), backend: True + **subdirectory.get(FIELDS.AVAILABLE_BACKENDS, {}), + backend: True, } if is_main: subdirectory[serializer.MAIN_FIELD] = True if results_dir is not None: subdirectory[serializer.RESULTS_DIR_FIELD] = results_dir - self._update_projection_geometry(subdirectory) + self._update_projection_geometry( + subdirectory, + (projection for projection, _path in retained_paths), + ) return data self._execute_update(metadata_path, update) @staticmethod - def _update_projection_geometry(subdirectory: dict[str, Any]) -> None: - entries = VirtualWorkspaceSourceProjectionEntries.from_subdirectory( - subdirectory - ).entries - if not entries: - return + def _update_projection_geometry( + subdirectory: dict[str, Any], + source_projections: Iterable[SourceProjection], + ) -> None: + """Derive geometry from the transaction's current nominal projections.""" unique_projections: dict[tuple[object, ...], SourceProjection] = {} - for projection in entries.values(): + for projection in source_projections: unique_projections.setdefault(projection.identity_key, projection) + if not unique_projections: + return projections = SourceProjectionSet(tuple(unique_projections.values())) subdirectory[FIELDS.GRID_DIMENSIONS] = ( SourceTileLayout.metadata_grid_dimensions(projections) @@ -674,51 +692,7 @@ def normalize_metadata_fields(metadata_fields: JsonValue) -> SourceMetadataMappi raise RuntimeError( "virtual_workspace source metadata values must be mappings." ) - return MappingProxyType( - { - str( - key - ): VirtualWorkspaceSourceMetadataEntries.normalize_metadata_value(value) - for key, value in metadata_fields.items() - } - ) - - @staticmethod - def normalize_metadata_value(value: JsonValue) -> SourceMetadataValue: - if isinstance(value, Mapping): - return MappingProxyType( - { - str( - nested_key - ): VirtualWorkspaceSourceMetadataEntries.require_scalar_metadata_value( - nested_value - ) - for nested_key, nested_value in value.items() - } - ) - return VirtualWorkspaceSourceMetadataEntries.require_scalar_metadata_value( - value - ) - - @staticmethod - def require_scalar_metadata_value(value: JsonValue) -> SourceMetadataScalar: - # Scalar fast path first: metadata values are overwhelmingly scalars, - # and the container ABC isinstance checks below are comparatively - # expensive per field. - if value is None or isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, Mapping) or ( - isinstance(value, Sequence) and not isinstance(value, str) - ): - raise RuntimeError( - "virtual_workspace source metadata supports scalar values and " - "one-level scalar mappings only." - ) - raise RuntimeError( - "virtual_workspace source metadata scalar values must be strings, " - "numbers, booleans, or null." - ) - return value + return DurableSourceMetadata.from_mapping(metadata_fields) def metadata_for(self, virtual_path: str) -> SourceMetadataMapping: metadata = self.entries.get(virtual_path) diff --git a/openhcs/interop/cellprofiler/analyst_export.py b/openhcs/interop/cellprofiler/analyst_export.py index 171e971eb4..db6c084f78 100644 --- a/openhcs/interop/cellprofiler/analyst_export.py +++ b/openhcs/interop/cellprofiler/analyst_export.py @@ -69,7 +69,7 @@ SourceImageProvenance, ) from openhcs.core.source_bindings import CompiledSourceBindingPlan -from openhcs.core.source_metadata import SourceMetadataRoleView, SourceMetadataScalar +from openhcs.core.source_metadata import SourceMetadataFields, SourceMetadataScalar from openhcs.core.source_workspace_projection import ( VirtualWorkspacePathLookup, VirtualWorkspaceSourceProjectionAuthority, @@ -955,7 +955,7 @@ def collect_image_provenance( ] = source_path.as_uri() if component_metadata is not None: metadata_items.update( - SourceMetadataRoleView(component_metadata).original_items() + SourceMetadataFields.original_items(component_metadata) ) image_number = self.image_set_numbering.for_source_slice( scope=scope, diff --git a/openhcs/interop/cellprofiler/measurement_lookup.py b/openhcs/interop/cellprofiler/measurement_lookup.py index 8d285990a6..120db857d4 100644 --- a/openhcs/interop/cellprofiler/measurement_lookup.py +++ b/openhcs/interop/cellprofiler/measurement_lookup.py @@ -9,6 +9,7 @@ from metaclass_registry import AutoRegisterMeta +from openhcs.core.runtime_relationships import ChildCountFeatureDeclaration from openhcs.core.public_api import declared_public_names from openhcs.core.measurement_feature_queries import measurement_values_for_feature @@ -151,19 +152,10 @@ class CellProfilerChildCountFeatureParser(CellProfilerMeasurementFeatureParser): kind = CellProfilerMeasurementFeatureKind.CHILD_COUNT kind_key = kind.value - prefix = "Children_" - suffix = "_Count" - def parse_feature( - self, - feature_name: str, - ) -> CellProfilerMeasurementFeature | None: - if not feature_name.startswith(self.prefix): - return None - if not feature_name.endswith(self.suffix): - return None - object_name = feature_name[len(self.prefix) : -len(self.suffix)].strip() - if not object_name: + def parse_feature(self, feature_name: str) -> CellProfilerMeasurementFeature | None: + object_name = ChildCountFeatureDeclaration.from_feature_name(feature_name) + if object_name is None: return None return CellProfilerMeasurementFeature( name=feature_name, @@ -172,18 +164,13 @@ def parse_feature( ) def feature_from_object_name( - self, - object_name: str, + self, object_name: str ) -> CellProfilerMeasurementFeature: - normalized = object_name.strip() - if not normalized: - raise ValueError( - "Child-count feature requires a non-empty child object name." - ) + feature_name = ChildCountFeatureDeclaration.feature_name(object_name) return CellProfilerMeasurementFeature( - name=f"{self.prefix}{normalized}{self.suffix}", + name=feature_name, kind=CellProfilerMeasurementFeatureKind.CHILD_COUNT, - object_name=normalized, + object_name=object_name.strip(), ) diff --git a/openhcs/interop/cellprofiler/runtime/artifact_binding.py b/openhcs/interop/cellprofiler/runtime/artifact_binding.py index 34015d04a9..55a279e1b8 100644 --- a/openhcs/interop/cellprofiler/runtime/artifact_binding.py +++ b/openhcs/interop/cellprofiler/runtime/artifact_binding.py @@ -181,6 +181,14 @@ def source_image_name( del request return None + def source_image_name_from_value( + self, + value: RuntimeCallableArgument, + ) -> str | None: + """Project a source name from an already resolved artifact value.""" + del value + return None + def source_image_payload( self, request: RuntimeArtifactInputRequest, @@ -253,11 +261,15 @@ def runtime_input_value( def source_image_name( self, request: RuntimeArtifactInputRequest, + ) -> str | None: + return self.source_image_name_from_value(self.raw_runtime_input_value(request)) + + def source_image_name_from_value( + self, + value: RuntimeCallableArgument, ) -> str | None: return single_source_name( - image_payload_metadata( - self.raw_runtime_input_value(request) - ).source_provenance.represented_source_image_names + image_payload_metadata(value).source_provenance.represented_source_image_names ) def source_image_payload( @@ -335,7 +347,13 @@ def source_image_name( self, request: RuntimeArtifactInputRequest, ) -> str | None: - return self.object_labels(request).source_image_name + return self.source_image_name_from_value(self.object_labels(request)) + + def source_image_name_from_value( + self, + value: RuntimeCallableArgument, + ) -> str | None: + return cast(ObjectLabelSet, value).source_image_name def source_image_payload( self, diff --git a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py index c1279ed9cd..199f2dea38 100644 --- a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py +++ b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py @@ -8,11 +8,9 @@ import numpy as np from openhcs.core.aligned_image_payload import ( - AlignedImageSliceContext, AlignedImageStack, ImagePayloadExecutionMode, aligned_image_stack_kwargs, - pack_aligned_image_outputs, ) from openhcs.core.callable_contract import CallableContract from openhcs.core.image_shapes import ArrayShape @@ -20,7 +18,6 @@ from openhcs.core.memory import detect_memory_type, stack_slices from openhcs.core.runtime_batch_contracts import ( Pure2DSliceBatchExecutor, - RuntimeBatchExecutionDomain, RuntimeBatchInvocationRequest, RuntimePure2DSliceBatchRequest, SliceIndexRuntimeParameter, @@ -30,7 +27,7 @@ image_payload_mask, image_payload_metadata, ) -from openhcs.core.runtime_output_matching import split_runtime_output +from openhcs.core.runtime_output_matching import RuntimeReturnedOutputMatcher from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, RuntimePlaneAxisValueProjection, @@ -59,10 +56,12 @@ RuntimeCallablePolicy, RuntimeCallableView, RuntimeInvocationKwargPolicy, + contextualize_main_image_output, ) _CELLPROFILER_RUNTIME_CALLABLE_POLICY = RuntimeCallablePolicy( - callable_view=RuntimeCallableView.RAW, + # execute() resolves the compiled raw target before processing dispatch. + callable_view=RuntimeCallableView.DECORATED, kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED, ) @@ -112,6 +111,7 @@ def execute( executor = type(self)(plane_projection=plane_projection) function_name = callable_contract.function_name processing_contract = callable_contract.require_processing_contract() + runtime_func = callable_contract.resolve_raw_runtime_callable() mode = execution_mode CellProfilerRuntimeProfileLogger.log_module_profile( "cp_executor_mode_resolution", @@ -131,12 +131,19 @@ def execute( | ProcessingContract.FLEXIBLE | ProcessingContract.VOLUMETRIC_TO_SLICE, ): + runtime_kwargs = dict(kwargs) + runtime_kwargs.update( + processing_contract.declaration.consume_semantic_controls( + runtime_kwargs, + parameters=callable_contract.canonical_signature.parameters, + ) + ) result = processing_contract.execute( executor, - func, + runtime_func, image, callable_contract=callable_contract, - **dict(kwargs), + **runtime_kwargs, ) case ( ImagePayloadExecutionMode.FULL_STACK, @@ -146,7 +153,7 @@ def execute( | ProcessingContract.VOLUMETRIC_TO_SLICE, ): result = executor.execute_pure_3d( - func, + runtime_func, image, callable_contract=callable_contract, **dict(kwargs), @@ -160,7 +167,7 @@ def execute( ): result = executor._execute_aligned_multi_image_stack( callable_contract, - func, + runtime_func, image, **dict(kwargs), ) @@ -176,79 +183,10 @@ def execute( function=function_name, mode=mode.value, ) - return executor._contextualize_multi_canonical_output( + return RuntimeReturnedOutputMatcher( callable_contract, result, - ) - - def _contextualize_multi_canonical_output( - self, - callable_contract: CallableContract, - result: RuntimeCallableArgument, - ) -> RuntimeCallableArgument: - """Attach the compiled canonical ABI to one exact returned image axis.""" - - canonical_specs = callable_contract.canonical_return_output_specs.specs - if len(canonical_specs) <= 1: - return result - canonical_output, trailing_outputs = split_runtime_output(result) - if isinstance(canonical_output, AlignedImageStack): - if canonical_output.slice_contexts: - return result - output_values = canonical_output.slices - else: - projection = self.plane_projection - function_name = callable_contract.function_name - if projection is None: - raise RuntimeSliceProjectionDeclarationError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs but returned a non-aligned payload without a " - "compiled plane projection." - ) - if projection.plane_index is not None: - raise RuntimeSliceProjectionDeclarationError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs after the compiled plane projection already selected " - f"plane {projection.plane_index}." - ) - if projection.axis_size != len(canonical_specs): - raise ValueError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs but its compiled plane projection declares " - f"{projection.axis_size} value(s)." - ) - output_axis = image_payload_metadata(canonical_output).plane_axis - if output_axis is not projection.axis: - raise RuntimeSliceProjectionDeclarationError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs but its returned payload does not declare the " - f"compiled {projection.axis.value!r} plane axis; got " - f"{output_axis!r}." - ) - output_values = tuple( - RuntimeSliceProjection.value_for_slice( - canonical_output, - projection.selected_plane(output_index), - ) - for output_index in range(projection.axis_size) - ) - if len(output_values) != len(canonical_specs): - raise ValueError( - f"{callable_contract.function_name} returned {len(output_values)} " - "canonical output value(s) for " - f"{len(canonical_specs)} compiled output spec(s)." - ) - contextualized_output = pack_aligned_image_outputs( - output_values, - slice_contexts=AlignedImageSliceContext.main_flow_for_artifact_specs( - canonical_specs - ), - ) - return ( - (contextualized_output, *trailing_outputs) - if trailing_outputs - else contextualized_output - ) + ).contextualize_canonical_output(plane_projection=executor.plane_projection) def execute_pure_2d_slice_batch( self, @@ -273,18 +211,12 @@ def execute_pure_2d_slice_batch( slices_2d=slices_2d, kwargs=kwargs, execute_slice=execute_slice, + signature=callable_contract.raw_runtime_signature, ) if slice_request.slice_count <= 0: return [], 0.0 - declared_batch_executor = callable_contract.runtime_batch_executor( - RuntimeBatchExecutionDomain.PURE_2D_SLICES - ) - batch_executor = ( - declared_batch_executor - if callable(declared_batch_executor) - else Pure2DSliceBatchExecutor.default_executor() - ) + batch_executor = Pure2DSliceBatchExecutor.from_executors(callable_contract.runtime_batch_executors) slice_started_at = time.perf_counter() if batch_executor is not None and slice_request.slice_count > 1: slice_results = list(batch_executor(slice_request)) @@ -323,9 +255,10 @@ def execute_pure_3d( ), ) call_started_at = time.perf_counter() - result = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + result = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( + callable_contract, func, - (projected_image,), + projected_image, projected_kwargs, ).call() CellProfilerRuntimeProfileLogger.log_module_profile( @@ -401,9 +334,10 @@ def execute_aligned_stack_slice( slice_index: int, slice_count: int, ) -> RuntimeCallableArgument: - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( + callable_contract, slice_func, - (slice_payload,), + slice_payload, aligned_image_stack_kwargs( slice_kwargs, slice_index, @@ -519,9 +453,10 @@ def execute_pure_2d( ) image_data = image_payload_data(image) if not isinstance(image_data, np.ndarray): - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( + callable_contract, func, - (image,), + image, kwargs, ).call() @@ -546,9 +481,10 @@ def execute_pure_2d( "declared plane projection. Kwargs cannot create image-axis " "execution semantics." ) - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( + callable_contract, func, - (image,), + image, kwargs, ).call() declared_plane_axis = image_payload_metadata(image).plane_axis @@ -650,10 +586,10 @@ def execute_volumetric_to_slice( callable_contract: CallableContract, **kwargs: RuntimeCallableArgument, ) -> RuntimeCallableArgument: - del callable_contract - result_2d = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + result_2d = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( + callable_contract, func, - (image,), + image, kwargs, ).call() result_data = image_payload_data(result_2d) @@ -683,11 +619,13 @@ def execute_pure_2d_slice( sliced_kwargs = dict(sliced_kwargs) slice_index_name = SliceIndexRuntimeParameter.require_parameter_name() sliced_kwargs[slice_index_name] = slice_index - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + result = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( + callable_contract, func, - (slice_2d,), + slice_2d, sliced_kwargs, ).call() + return contextualize_main_image_output(slice_2d, result) def slice_pure_2d_kwargs( self, diff --git a/openhcs/interop/cellprofiler/runtime/module_execution.py b/openhcs/interop/cellprofiler/runtime/module_execution.py index da914bbcb4..170b24ac37 100644 --- a/openhcs/interop/cellprofiler/runtime/module_execution.py +++ b/openhcs/interop/cellprofiler/runtime/module_execution.py @@ -1323,6 +1323,7 @@ def _image_request( adapter, ), ) + image_input_specs = ArtifactSpecCollection(image_inputs) payloads = [] source_names: list[str | None] = [] image_strategy = RuntimeArtifactTypeStrategy.for_artifact_type( @@ -1331,8 +1332,11 @@ def _image_request( input_binding = replace(input_binding, current_image=current_runtime_payload) for spec in image_inputs: request = input_binding.artifact_request_for_spec(spec) - payloads.append(image_strategy.runtime_input_value(request)) - source_names.append(image_strategy.source_image_name(request)) + payload = image_strategy.runtime_input_value(request) + payloads.append(payload) + source_names.append( + image_strategy.source_image_name_from_value(payload) + ) parameter_image_inputs = input_binding.image_inputs broadcast_sources = tuple( sources[0] @@ -1358,8 +1362,8 @@ def _image_request( if align_primary_images else () ), - stack_broadcast_source_indices=self._stack_broadcast_source_indices( - image_inputs + stack_broadcast_source_indices=( + image_input_specs.stack_broadcast_source_indices() ), ) return CellProfilerImageRequest( @@ -1367,12 +1371,12 @@ def _image_request( source_image_name=self._primary_image_source_name_from_sources( image_inputs, tuple(source_names) ), - source_aliases=ArtifactSpecCollection(image_inputs).names(), + source_aliases=image_input_specs.names(), image_count=len(payloads), execution_mode=composition.execution_mode, plane_projection=composition.preserved_plane_projection( adapter, - source_aliases=ArtifactSpecCollection(image_inputs).names(), + source_aliases=image_input_specs.names(), ), ) @@ -1409,40 +1413,6 @@ def _primary_image_source_name_from_sources( return None return source_names[0] - @staticmethod - def _stack_broadcast_source_indices( - input_specs: tuple[ArtifactSpec, ...], - ) -> tuple[int | None, ...]: - """Resolve exact stack-broadcast owners from declared input relations.""" - - indices_by_ref: dict[ArtifactSpecRef, list[int]] = {} - for input_index, spec in enumerate(input_specs): - indices_by_ref.setdefault(spec.ref(), []).append(input_index) - - result: list[int | None] = [] - for input_index, spec in enumerate(input_specs): - sources = spec.stack_broadcast_sources() - if len(sources) > 1: - raise ValueError( - f"Input {spec.ref()!r} declares multiple stack-broadcast " - f"owners: {sources!r}." - ) - if not sources: - result.append(None) - continue - source_indices = tuple(indices_by_ref.get(sources[0], ())) - if len(source_indices) != 1: - raise ValueError( - f"Input {spec.ref()!r} requires exactly one active occurrence " - f"of stack-broadcast owner {sources[0]!r}, got " - f"{source_indices!r}." - ) - source_index = source_indices[0] - if source_index == input_index: - raise ValueError(f"Input {spec.ref()!r} cannot broadcast from itself.") - result.append(source_index) - return tuple(result) - def _invocation_request( self, *, diff --git a/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py b/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py index f34f814108..5849251569 100644 --- a/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py +++ b/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py @@ -17,8 +17,6 @@ ) from openhcs.core.runtime_measurements import ( MeasurementRowAxisField, - ObjectReferenceFeatureMarker, - RuntimeMeasurementFeatureDeclaration, ) from openhcs.core.runtime_object_label_domains import ( ObjectLabelDomainScope, @@ -35,6 +33,8 @@ from openhcs.core.runtime_relationships import ( ObjectRelationship, ObjectRelationshipDeclaration, + DirectParentReferenceFeatureDeclaration, + DirectParentReferenceMeasurementFeature, ) from openhcs.core.runtime_image_values import ( image_payload_metadata, @@ -53,50 +53,6 @@ ) -class DirectParentReferenceFeatureMarker(ObjectReferenceFeatureMarker): - """Semantic marker for a child's direct parent-object reference.""" - - -@dataclass(frozen=True, slots=True) -class DirectParentReferenceMeasurementFeature: - """Nominal identity encoded by a ``Parent_`` measurement name.""" - - parent_object_name: str - - def __post_init__(self) -> None: - if not isinstance(self.parent_object_name, str) or not self.parent_object_name: - raise ValueError("Direct parent-reference object name cannot be empty.") - - -class DirectParentReferenceFeatureDeclaration(RuntimeMeasurementFeatureDeclaration): - """Parse and render direct parent references at their row-production owner.""" - - declaration_key = "direct_parent_reference" - semantic_marker_types = (DirectParentReferenceFeatureMarker,) - prefix = "Parent_" - - @classmethod - def from_feature_name( - cls, - feature_name: str, - ) -> DirectParentReferenceMeasurementFeature | None: - if not feature_name.startswith(cls.prefix): - return None - parent_object_name = feature_name[len(cls.prefix) :] - if not parent_object_name: - return None - return DirectParentReferenceMeasurementFeature(parent_object_name) - - @classmethod - def feature_name(cls, identity: object) -> str: - if not isinstance(identity, DirectParentReferenceMeasurementFeature): - raise TypeError( - f"{cls.__name__}.feature_name requires " - "DirectParentReferenceMeasurementFeature." - ) - return f"{cls.prefix}{identity.parent_object_name}" - - @dataclass(frozen=True, slots=True) class RelationshipMeasurementRows: """Project parent-child relationship payloads into CP object measurement rows.""" diff --git a/openhcs/processing/backends/cellprofiler/_backend.py b/openhcs/processing/backends/cellprofiler/_backend.py index e37742d5b1..811a6deebc 100644 --- a/openhcs/processing/backends/cellprofiler/_backend.py +++ b/openhcs/processing/backends/cellprofiler/_backend.py @@ -14,6 +14,7 @@ from openhcs.constants.constants import MemoryType from openhcs.core.callable_contract import CallableContract from openhcs.core.processing_preparation import PersistentNumbaKernelPreparation +from openhcs.core.runtime_object_labels import DenseArrayObjectLabelStorageStrategy from openhcs.core.runtime_plane_projection import RuntimeSliceInvariantValue @@ -306,6 +307,7 @@ def prepare_registered_family(cls) -> None: """Prepare every registered backend implementation for compiler warmup.""" if cls is CellProfilerBackendStrategyMixin: return + DenseArrayObjectLabelStorageStrategy.prepare_coordinates() snapshot = CellProfilerBackendRegistrySnapshot.for_family( cls, MemoryType.NUMPY, diff --git a/openhcs/processing/backends/cellprofiler/_preparation.py b/openhcs/processing/backends/cellprofiler/_preparation.py index 933a8a3795..cea2850ff1 100644 --- a/openhcs/processing/backends/cellprofiler/_preparation.py +++ b/openhcs/processing/backends/cellprofiler/_preparation.py @@ -3,6 +3,7 @@ from __future__ import annotations from openhcs.core.processing_preparation import RegisteredNumbaKernelPreparation +from openhcs.core.runtime_object_labels import DenseArrayObjectLabelStorageStrategy from openhcs.processing.backends.cellprofiler.perf_fixtures import capture_enabled @@ -20,4 +21,5 @@ def can_prepare_in_child(cls) -> bool: def prepare_registered_family(cls) -> None: if capture_enabled(): return + DenseArrayObjectLabelStorageStrategy.prepare_coordinates() super().prepare_registered_family() diff --git a/openhcs/processing/backends/cellprofiler/alignment.py b/openhcs/processing/backends/cellprofiler/alignment.py index 9ef9b11ba2..b32e4a5ef5 100644 --- a/openhcs/processing/backends/cellprofiler/alignment.py +++ b/openhcs/processing/backends/cellprofiler/alignment.py @@ -40,6 +40,7 @@ RuntimePlaneAxisValueProjection, ) from openhcs.core.runtime_slice_projection import RuntimeSliceProjection +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, MaskedImagePayload, @@ -1083,7 +1084,7 @@ def prepare_align() -> None: @required_variable_components(VariableComponents.CHANNEL) @numpy(contract=ProcessingContract.PURE_3D) def align( - image: np.ndarray, + image: RuntimeArrayData, *, method: AlignModule.Method = AlignModule.Method.MUTUAL_INFORMATION, crop_mode: AlignModule.CropMode = AlignModule.CropMode.KEEP_SIZE, diff --git a/openhcs/processing/backends/cellprofiler/area_occupied.py b/openhcs/processing/backends/cellprofiler/area_occupied.py index 44049e52b8..e748871e96 100644 --- a/openhcs/processing/backends/cellprofiler/area_occupied.py +++ b/openhcs/processing/backends/cellprofiler/area_occupied.py @@ -728,7 +728,7 @@ def measure(self) -> AreaOccupiedMeasurement: SliceIndexRuntimeParameter, ) def measure_image_area_occupied( - image: np.ndarray, + image: RuntimeArrayData, *, operand_choices: Sequence[OperandChoice] = (OperandChoice.BINARY_IMAGE,), area_occupied_rows: Sequence[AreaOccupiedRow] = (), diff --git a/openhcs/processing/backends/cellprofiler/classification.py b/openhcs/processing/backends/cellprofiler/classification.py index 4783e7e795..e7e22bd10a 100644 --- a/openhcs/processing/backends/cellprofiler/classification.py +++ b/openhcs/processing/backends/cellprofiler/classification.py @@ -2028,7 +2028,7 @@ def object_classification_backend( SliceIndexRuntimeParameter, ) def classify_objects_single_measurement( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, measurement_feature: str = "", measurement_values: np.ndarray | None = None, diff --git a/openhcs/processing/backends/cellprofiler/colocalization.py b/openhcs/processing/backends/cellprofiler/colocalization.py index 674a89855e..9c5d6dfef1 100644 --- a/openhcs/processing/backends/cellprofiler/colocalization.py +++ b/openhcs/processing/backends/cellprofiler/colocalization.py @@ -2522,7 +2522,7 @@ def _colocalization_unit_interval_scale( @numpy(contract=ProcessingContract.FLEXIBLE) @runtime_bound_parameters(_ColocalizationThresholdMaskOutputsRuntimeParameter) def measure_colocalization( - image: np.ndarray, + image: RuntimeArrayData, channel_1: int = 0, channel_2: int = 1, threshold_percent: float = 15.0, @@ -2790,7 +2790,7 @@ def _measure_colocalization_objects_core( _ColocalizationCostesThresholdBatchRuntimeParameter, ) def measure_colocalization_objects( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, measurement_scope: CellProfilerMeasurementTargetScope = CellProfilerMeasurementTargetScope.OBJECT, channel_1: int = 0, @@ -2881,7 +2881,7 @@ def measure_colocalization_objects( def _colocalization_threshold_mask_canonical_output( - image: np.ndarray, + image: RuntimeArrayData, *, threshold_mask_groups: tuple[ColocalizationThresholdMaskGroup, ...], threshold_mask_outputs: tuple[ColocalizationThresholdMaskRuntimeOutput, ...], @@ -2911,7 +2911,7 @@ def _colocalization_threshold_mask_canonical_output( def _colocalization_threshold_mask( - image: np.ndarray, + image: RuntimeArrayData, request: ColocalizationThresholdMaskRuntimeOutput, ) -> RuntimeArrayData: """Apply CellProfiler's whole-image or per-object percentage threshold.""" diff --git a/openhcs/processing/backends/cellprofiler/color.py b/openhcs/processing/backends/cellprofiler/color.py index a762c62592..16ebc86a8e 100644 --- a/openhcs/processing/backends/cellprofiler/color.py +++ b/openhcs/processing/backends/cellprofiler/color.py @@ -1542,7 +1542,7 @@ def run(self, request: GrayToColorRequest) -> np.ndarray: @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def gray_to_color( - image: np.ndarray, + image: RuntimeArrayData, color_scheme: GrayToColorModule.Scheme = GrayToColorModule.Scheme.RGB, rescale_intensity: bool = True, red_channel: int = -1, @@ -1641,7 +1641,7 @@ def gray_to_color( ) @numpy(contract=ProcessingContract.FLEXIBLE) def color_to_gray( - image: np.ndarray, + image: RuntimeArrayData, mode: ColorToGrayMode = ColorToGrayMode.SPLIT, image_type: ImageChannelType = ImageChannelType.RGB, channel_indices: tuple[int, ...] = ColorToGrayModule.default_channel_indices, @@ -1673,7 +1673,7 @@ def color_to_gray( def _invert_for_printing_channels( - image: np.ndarray, + image: RuntimeArrayData, *, input_mode: InvertInputMode, use_red_input: bool, @@ -1715,7 +1715,7 @@ def _invert_for_printing_channels( def _invert_for_printing_result( - image: np.ndarray, + image: RuntimeArrayData, *, input_mode: InvertInputMode, use_red_input: bool, @@ -1797,7 +1797,7 @@ def _invert_for_printing_result( @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def invert_for_printing( - image: np.ndarray, + image: RuntimeArrayData, input_mode: InvertInputMode = InvertInputMode.COLOR, use_red_input: bool = True, use_green_input: bool = True, @@ -1833,7 +1833,7 @@ def invert_for_printing( @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def invert_for_printing_grayscale( - image: np.ndarray, + image: RuntimeArrayData, input_mode: InvertInputMode = InvertInputMode.COLOR, use_red_input: bool = True, use_green_input: bool = True, @@ -1876,7 +1876,7 @@ def invert_for_printing_grayscale( @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def invert_for_printing_without_output( - image: np.ndarray, + image: RuntimeArrayData, input_mode: InvertInputMode = InvertInputMode.COLOR, use_red_input: bool = True, use_green_input: bool = True, @@ -1915,7 +1915,7 @@ def invert_for_printing_without_output( def combine_color_to_gray( - image: np.ndarray, + image: RuntimeArrayData, channel_indices: tuple[int, ...], contributions: tuple[float, ...], ) -> np.ndarray: @@ -1931,7 +1931,7 @@ def combine_color_to_gray( return restore_color_to_gray_shape(image, result) -def color_to_gray_combine_output_metadata(image: np.ndarray): +def color_to_gray_combine_output_metadata(image: RuntimeArrayData): """Return metadata for a color-to-grayscale semantic collapse.""" return ( image_payload_metadata(image) @@ -1941,7 +1941,7 @@ def color_to_gray_combine_output_metadata(image: np.ndarray): def split_color_to_gray( - image: np.ndarray, image_type: ImageChannelType, channel_indices: tuple[int, ...] + image: RuntimeArrayData, image_type: ImageChannelType, channel_indices: tuple[int, ...] ) -> tuple[np.ndarray, ...]: color_stack = nhwc_color_stack(image).astype(np.float32) source_stack = ( @@ -1967,7 +1967,7 @@ def color_to_gray_channel(color_stack: np.ndarray, channel_index: int) -> np.nda return color_stack[..., channel_index] -def nhwc_color_stack(image: np.ndarray) -> np.ndarray: +def nhwc_color_stack(image: RuntimeArrayData) -> np.ndarray: """Return NHWC pixels from explicitly declared image layout metadata.""" image_data = np.asarray(image_payload_data(image)) metadata = image_payload_metadata(image) @@ -1998,7 +1998,7 @@ def nhwc_color_stack(image: np.ndarray) -> np.ndarray: return channel_last -def restore_color_to_gray_shape(original: np.ndarray, stack: np.ndarray) -> np.ndarray: +def restore_color_to_gray_shape(original: RuntimeArrayData, stack: np.ndarray) -> np.ndarray: metadata = image_payload_metadata(original) if metadata.is_declared_source_channel_plane(original): if stack.shape[0] != 1: @@ -2056,7 +2056,7 @@ def rgb_to_hsv_stack(rgb_stack: np.ndarray) -> np.ndarray: @numpy(contract=ProcessingContract.FLEXIBLE) def unmix_colors( - image: np.ndarray, + image: RuntimeArrayData, stain_names: Sequence[StainType] = (), custom_absorbances: Sequence[Sequence[float] | None] = (), stain1: StainType = StainType.HEMATOXYLIN, diff --git a/openhcs/processing/backends/cellprofiler/crop.py b/openhcs/processing/backends/cellprofiler/crop.py index 62fd7e1ba9..79db9fe1e5 100644 --- a/openhcs/processing/backends/cellprofiler/crop.py +++ b/openhcs/processing/backends/cellprofiler/crop.py @@ -24,6 +24,7 @@ ) from openhcs.core.pipeline.function_contracts import special_inputs from openhcs.core.runtime_measurements import RuntimeMeasurementFeature +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, @@ -608,7 +609,7 @@ def cropped_pixels(self) -> np.ndarray: class CropRequest: """Executable CellProfiler Crop request.""" - image: np.ndarray + image: RuntimeArrayData mask_plane: np.ndarray | None = None crop_shape: CropModule.Shape = CropModule.Shape.RECTANGLE cropping_method: CropModule.Method = CropModule.Method.COORDINATES @@ -857,7 +858,7 @@ def crop_output_metadata( @numpy_decorator(contract=ProcessingContract.PURE_2D) @special_inputs("topology_inputs") def crop( - image: np.ndarray, + image: RuntimeArrayData, topology_inputs: tuple[np.ndarray | ObjectLabelValue, ...] = (), crop_shape: CropModule.Shape = CropModule.Shape.RECTANGLE, cropping_method: CropModule.Method = CropModule.Method.COORDINATES, diff --git a/openhcs/processing/backends/cellprofiler/display_modules.py b/openhcs/processing/backends/cellprofiler/display_modules.py index 28c40ce4f4..d5566f0a94 100644 --- a/openhcs/processing/backends/cellprofiler/display_modules.py +++ b/openhcs/processing/backends/cellprofiler/display_modules.py @@ -29,6 +29,7 @@ special_inputs, ) from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_metadata, @@ -492,7 +493,7 @@ def render_slice(self) -> np.ndarray: @special_inputs("labels") @runtime_bound_parameters(_DisplayMeasurementsRuntimeParameter) def display_data_on_image( - image: np.ndarray, + image: RuntimeArrayData, labels: Optional[ObjectLabelValue] = None, measurements: Optional[np.ndarray] = None, measurement_feature: Optional[str] = None, diff --git a/openhcs/processing/backends/cellprofiler/edge.py b/openhcs/processing/backends/cellprofiler/edge.py index 30bc44280d..5e01b30a19 100644 --- a/openhcs/processing/backends/cellprofiler/edge.py +++ b/openhcs/processing/backends/cellprofiler/edge.py @@ -50,6 +50,7 @@ class EnhanceEdgesModule(CellProfilerModule): from numba import njit from openhcs.core.memory.decorators import numpy as numpy_decorator from openhcs.core.public_api import public_names_from_objects +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_mask, @@ -508,7 +509,7 @@ def _native_kirsch(image: np.ndarray) -> np.ndarray: @numpy_decorator(contract=ProcessingContract.PURE_2D) def enhance_edges( - image: np.ndarray, + image: RuntimeArrayData, method: EdgeMethod = EdgeMethod.SOBEL, direction: EdgeDirection = EdgeDirection.ALL, edge_backend_provider: BackendProviderInput = DEFAULT_CELLPROFILER_BACKEND_SELECTION, diff --git a/openhcs/processing/backends/cellprofiler/feature_enhancement.py b/openhcs/processing/backends/cellprofiler/feature_enhancement.py index b8f6bd7bda..a2b813799b 100644 --- a/openhcs/processing/backends/cellprofiler/feature_enhancement.py +++ b/openhcs/processing/backends/cellprofiler/feature_enhancement.py @@ -13,6 +13,7 @@ from openhcs.core.callable_contract import processing_prepare from openhcs.core.memory.decorators import numpy from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_mask, @@ -65,7 +66,7 @@ class NeuriteMethod(Enum): @numpy(contract=ProcessingContract.PURE_2D) def enhance_or_suppress_features( - image: np.ndarray, + image: RuntimeArrayData, method: OperationMethod = OperationMethod.ENHANCE, enhance_method: EnhanceMethod = EnhanceMethod.SPECKLES, radius: float = 10.0, diff --git a/openhcs/processing/backends/cellprofiler/gaussian_filter.py b/openhcs/processing/backends/cellprofiler/gaussian_filter.py index 187d4d972d..7018d62595 100644 --- a/openhcs/processing/backends/cellprofiler/gaussian_filter.py +++ b/openhcs/processing/backends/cellprofiler/gaussian_filter.py @@ -7,6 +7,7 @@ from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode from openhcs.core.callable_contract import runtime_image_execution_mode +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_metadata, @@ -41,7 +42,7 @@ class GaussianFilterModule(CellProfilerModule): @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy(contract=ProcessingContract.FLEXIBLE) -def gaussian_filter(image: np.ndarray, sigma: float = 1.0) -> np.ndarray: +def gaussian_filter(image: RuntimeArrayData, sigma: float = 1.0) -> np.ndarray: """ Apply CellProfiler-compatible Gaussian smoothing to an image. diff --git a/openhcs/processing/backends/cellprofiler/grid.py b/openhcs/processing/backends/cellprofiler/grid.py index ad92c9b044..3c83012bde 100644 --- a/openhcs/processing/backends/cellprofiler/grid.py +++ b/openhcs/processing/backends/cellprofiler/grid.py @@ -18,6 +18,7 @@ from openhcs.constants.constants import VariableComponents from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.artifacts import ( ArtifactSpecCollection, ArtifactSpecRelation, @@ -1293,14 +1294,14 @@ class IdentifyObjectsInGridRequest(GridShapeContext): """Executable request for CellProfiler IdentifyObjectsInGrid semantics.""" registry_key = "identify_objects" - image: np.ndarray + image: RuntimeArrayData shape_choice: ShapeChoice @classmethod def from_runtime( cls, *, - image: np.ndarray, + image: RuntimeArrayData, grid_definition: GridRuntimeDefinitionRequest, shape_choice: ShapeChoice, diameter_choice: DiameterChoice, @@ -1485,7 +1486,7 @@ def draw_grid_overlay( @numpy(contract=ProcessingContract.PURE_2D) @special_inputs("topology_inputs") def identify_objects_in_grid( - image: np.ndarray, + image: RuntimeArrayData, topology_inputs: tuple[SpatialGrid | ObjectLabelValue, ...], grid_rows: int = 8, grid_columns: int = 12, diff --git a/openhcs/processing/backends/cellprofiler/illumination.py b/openhcs/processing/backends/cellprofiler/illumination.py index db283789d6..3c53edf1f8 100644 --- a/openhcs/processing/backends/cellprofiler/illumination.py +++ b/openhcs/processing/backends/cellprofiler/illumination.py @@ -922,7 +922,7 @@ def image_payload( @numpy(contract=ProcessingContract.FLEXIBLE) def correct_illumination_calculate( - image: np.ndarray, + image: RuntimeArrayData, intensity_choice: IntensityChoice = IntensityChoice.REGULAR, dilate_objects: bool = False, object_dilation_radius: int = 1, diff --git a/openhcs/processing/backends/cellprofiler/image_geometry.py b/openhcs/processing/backends/cellprofiler/image_geometry.py index 61bdddcca5..f51560c8a8 100644 --- a/openhcs/processing/backends/cellprofiler/image_geometry.py +++ b/openhcs/processing/backends/cellprofiler/image_geometry.py @@ -1093,12 +1093,12 @@ def align_label_plane_to_shape( @numpy(contract=ProcessingContract.FLEXIBLE) @special_inputs("mask") def mask_image( - image: np.ndarray, + image: RuntimeArrayData, mask: np.ndarray | ObjectLabelValue, mask_source: MaskSource = MaskSource.IMAGE, invert_mask: bool = False, binary_threshold: float = 0.5, -) -> np.ndarray: +) -> RuntimeArrayData: """Mask an image using CellProfiler image/object mask semantics. Args: @@ -1130,7 +1130,7 @@ def mask_image( def masked_image_plane( - image: np.ndarray, binary_mask: np.ndarray, *, invert_mask: bool + image: RuntimeArrayData, binary_mask: np.ndarray, *, invert_mask: bool ) -> tuple[np.ndarray, np.ndarray]: image_data = image_payload_data(image) if invert_mask: @@ -1159,7 +1159,7 @@ def masked_image_plane( @numpy(contract=ProcessingContract.PURE_2D) def mask_image_with_binary( - image: np.ndarray, invert_mask: bool = False + image: RuntimeArrayData, invert_mask: bool = False ) -> RuntimeArrayData: """Return a binary mask plane, optionally inverted.""" binary_mask = image_payload_data(image) > 0.5 @@ -1220,7 +1220,7 @@ def tile_output_shape( @numpy(contract=ProcessingContract.FLEXIBLE) def tile( - image: np.ndarray, + image: RuntimeArrayData, rows: int = 8, columns: int = 12, place_first: PlaceFirst = PlaceFirst.TOP_LEFT, @@ -1276,14 +1276,14 @@ def tile( @numpy(contract=ProcessingContract.PURE_2D) def resize( - image: np.ndarray, + image: RuntimeArrayData, resize_method: ResizeMethod = ResizeMethod.BY_FACTOR, resizing_factor_x: float = 0.25, resizing_factor_y: float = 0.25, specific_width: int = 100, specific_height: int = 100, interpolation: InterpolationMethod = InterpolationMethod.NEAREST_NEIGHBOR, -) -> np.ndarray: +) -> RuntimeArrayData: """Resize a CellProfiler image plane by factor or explicit dimensions.""" pixels = image_payload_data(image) geometry = ResizeGeometry.from_parameters( @@ -1298,7 +1298,7 @@ def resize( @numpy(contract=ProcessingContract.PURE_3D) def resize_volumetric( - image: np.ndarray, + image: RuntimeArrayData, resize_method: ResizeMethod = ResizeMethod.BY_FACTOR, resizing_factor_x: float = 0.25, resizing_factor_y: float = 0.25, @@ -1307,7 +1307,7 @@ def resize_volumetric( specific_height: int = 100, specific_planes: int = 10, interpolation: InterpolationMethod = InterpolationMethod.NEAREST_NEIGHBOR, -) -> np.ndarray: +) -> RuntimeArrayData: """Resize a CellProfiler ZYX image volume by factor or explicit dimensions.""" pixels = image_payload_data(image) geometry = ResizeGeometry.from_trailing_spatial_parameters( @@ -1322,7 +1322,7 @@ def resize_volumetric( @numpy(contract=ProcessingContract.PURE_2D) def flip_and_rotate( - image: np.ndarray, + image: RuntimeArrayData, flip_method: FlipMethod = FlipMethod.NONE, rotate_method: RotateMethod = RotateMethod.NONE, rotation_angle: float = 0.0, diff --git a/openhcs/processing/backends/cellprofiler/image_quality.py b/openhcs/processing/backends/cellprofiler/image_quality.py index e4338ba2f8..741d26e074 100644 --- a/openhcs/processing/backends/cellprofiler/image_quality.py +++ b/openhcs/processing/backends/cellprofiler/image_quality.py @@ -2,6 +2,7 @@ from __future__ import annotations +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.process_local_cache import RegisteredProcessLocalBoundedCache from abc import ABC, abstractmethod @@ -1565,7 +1566,7 @@ def _haralick_h3_numba(image: np.ndarray, scale: int) -> float: @numpy(contract=ProcessingContract.PURE_2D) def measure_image_quality( - image: np.ndarray, + image: RuntimeArrayData, include_scaling: bool = True, calculate_blur: bool = True, calculate_saturation: bool = True, diff --git a/openhcs/processing/backends/cellprofiler/intensity.py b/openhcs/processing/backends/cellprofiler/intensity.py index 7575aba5f7..f7bed622bf 100644 --- a/openhcs/processing/backends/cellprofiler/intensity.py +++ b/openhcs/processing/backends/cellprofiler/intensity.py @@ -303,6 +303,7 @@ def active_artifact_bindings( from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImageMetadataPayload, MaskedImagePayload, @@ -1560,7 +1561,7 @@ def _object_intensity_batch_key( @object_label_input_execution_mode(ObjectLabelInputExecutionMode.SLICE_ALIGNED) @runtime_bound_parameters(SliceIndexRuntimeParameter) def measure_object_intensity( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, object_intensity_backend_provider: BackendProviderInput = DEFAULT_CELLPROFILER_BACKEND_SELECTION, slice_index: int = OBJECT_INTENSITY_DEFAULT_SLICE_INDEX, @@ -1857,7 +1858,7 @@ def run(self, context: RescaleIntensityContext) -> np.ndarray: @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.PURE_2D) def rescale_intensity( - image: np.ndarray, + image: RuntimeArrayData, rescale_method: RescaleMethod = RescaleMethod.STRETCH, automatic_low: AutomaticLow = AutomaticLow.EACH_IMAGE, automatic_high: AutomaticHigh = AutomaticHigh.EACH_IMAGE, diff --git a/openhcs/processing/backends/cellprofiler/intensity_distribution.py b/openhcs/processing/backends/cellprofiler/intensity_distribution.py index c9c89ba06d..75d78d4341 100644 --- a/openhcs/processing/backends/cellprofiler/intensity_distribution.py +++ b/openhcs/processing/backends/cellprofiler/intensity_distribution.py @@ -2461,7 +2461,7 @@ def _radial_distribution_arrays_from_bin_totals_numba( _IntensityDistributionHeatmapOutputsRuntimeParameter, ) def measure_object_intensity_distribution( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, bin_count: int = 4, wants_scaled: bool = True, @@ -2562,7 +2562,7 @@ def measure_object_intensity_distribution( def _intensity_distribution_heatmap( - image: np.ndarray, + image: RuntimeArrayData, request: IntensityDistributionHeatmapRuntimeOutput, *, radial_backend: "RadialDistributionBackendStrategy", diff --git a/openhcs/processing/backends/cellprofiler/manual_objects.py b/openhcs/processing/backends/cellprofiler/manual_objects.py index f4c64435b8..6ae87fa586 100644 --- a/openhcs/processing/backends/cellprofiler/manual_objects.py +++ b/openhcs/processing/backends/cellprofiler/manual_objects.py @@ -16,6 +16,7 @@ import numpy as np +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.artifacts import ( ArtifactSpec, ImageArtifactType, @@ -63,7 +64,7 @@ class ManualObjectStats: ) @numpy(contract=ProcessingContract.PURE_2D) def identify_objects_manually( - image: np.ndarray, + image: RuntimeArrayData, labels_input: np.ndarray | None = None, ) -> tuple[ np.ndarray, diff --git a/openhcs/processing/backends/cellprofiler/median_filter.py b/openhcs/processing/backends/cellprofiler/median_filter.py index ca24bd2c00..ec19edfcc9 100644 --- a/openhcs/processing/backends/cellprofiler/median_filter.py +++ b/openhcs/processing/backends/cellprofiler/median_filter.py @@ -19,6 +19,7 @@ RuntimePure2DSliceBatchRequest, pure_2d_batch_executor, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, with_image_payload_data, @@ -371,7 +372,7 @@ def median_filter_backend( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy(contract=ProcessingContract.FLEXIBLE) def medianfilter( - image: np.ndarray, + image: RuntimeArrayData, window_size: int = 3, mode: ScipyBoundaryMode = ScipyBoundaryMode.CONSTANT, ) -> np.ndarray: diff --git a/openhcs/processing/backends/cellprofiler/morphology.py b/openhcs/processing/backends/cellprofiler/morphology.py index e79df88ea6..eeff8a0b25 100644 --- a/openhcs/processing/backends/cellprofiler/morphology.py +++ b/openhcs/processing/backends/cellprofiler/morphology.py @@ -690,6 +690,7 @@ def artifact_output_relations( object_label_identity_lineage_payload, object_label_parent_child_payload, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, @@ -1491,7 +1492,7 @@ def _morph_image_pixels( def _morph_image_payload( - image: np.ndarray, + image: RuntimeArrayData, structuring_element: StructuringElement, size: int, operation: Callable[[np.ndarray, np.ndarray], np.ndarray], @@ -1508,7 +1509,7 @@ def _morph_image_payload( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.FLEXIBLE) def closing( - image: np.ndarray, + image: RuntimeArrayData, structuring_element: StructuringElementInput = StructuringElement.DISK, size: StructuringElementSize = 3, morphology_backend_provider: BackendProviderInput = CellProfilerBackendProvider.NATIVE, @@ -1530,7 +1531,7 @@ def closing( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.FLEXIBLE) def opening( - image: np.ndarray, + image: RuntimeArrayData, structuring_element: StructuringElementInput = StructuringElement.DISK, size: StructuringElementSize = 3, morphology_backend_provider: BackendProviderInput = CellProfilerBackendProvider.NATIVE, diff --git a/openhcs/processing/backends/cellprofiler/object_images.py b/openhcs/processing/backends/cellprofiler/object_images.py index 2ec98235ff..81912cc368 100644 --- a/openhcs/processing/backends/cellprofiler/object_images.py +++ b/openhcs/processing/backends/cellprofiler/object_images.py @@ -10,6 +10,7 @@ from metaclass_registry import AutoRegisterMeta import numpy as np +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.artifacts import ImageArtifactType, ObjectLabelsArtifactType from openhcs.core.memory import numpy as numpy_decorator from openhcs.core.measurement_row_materialization import ( @@ -181,7 +182,7 @@ def object_label_colormap(colormap_name: str, num_labels: int) -> np.ndarray: @numpy_decorator(contract=ProcessingContract.PURE_2D) def convert_image_to_objects( - image: np.ndarray, + image: RuntimeArrayData, cast_to_bool: bool = False, preserve_label: bool = False, background: int = 0, diff --git a/openhcs/processing/backends/cellprofiler/outlines.py b/openhcs/processing/backends/cellprofiler/outlines.py index cdaeb5cefd..d0e4791fe7 100644 --- a/openhcs/processing/backends/cellprofiler/outlines.py +++ b/openhcs/processing/backends/cellprofiler/outlines.py @@ -878,7 +878,7 @@ def render_single_plane(self, image_sources: tuple[np.ndarray, ...]) -> np.ndarr @object_label_input_execution_mode(ObjectLabelInputExecutionMode.MATCH_IMAGE_STACK) @special_inputs("object_labels") def overlay_outlines( - image: np.ndarray, + image: RuntimeArrayData, *, blank_image: bool = False, display_mode: OutlineDisplayMode = OutlineDisplayMode.COLOR, @@ -922,7 +922,7 @@ def overlay_outlines( @object_label_input_execution_mode(ObjectLabelInputExecutionMode.MATCH_IMAGE_STACK) @special_inputs("labels") def overlay_objects( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, opacity: float = 0.3, max_label: int | None = None, @@ -1000,7 +1000,7 @@ def overlay_objects( def _overlay_objects_array( - image: np.ndarray, + image: RuntimeArrayData, labels: np.ndarray, *, opacity: float, @@ -1181,7 +1181,7 @@ def _draw_object_labels( def _draw_outline_image( output: np.ndarray, - outline_image: np.ndarray, + outline_image: RuntimeArrayData, color: tuple[float, float, float], *, outline_intensity: float, @@ -1200,7 +1200,7 @@ def _draw_outline_image( return output -def _outline_image_mask(outline_image: np.ndarray) -> np.ndarray: +def _outline_image_mask(outline_image: RuntimeArrayData) -> np.ndarray: mask = np.asarray(image_payload_data(outline_image)) > 0 channel_axis = image_payload_metadata(outline_image).normalized_source_channel_axis( outline_image diff --git a/openhcs/processing/backends/cellprofiler/primary_objects.py b/openhcs/processing/backends/cellprofiler/primary_objects.py index 70c083129a..9f51d8e0ba 100644 --- a/openhcs/processing/backends/cellprofiler/primary_objects.py +++ b/openhcs/processing/backends/cellprofiler/primary_objects.py @@ -11,6 +11,7 @@ from enum import Enum from openhcs.core.memory import numpy from openhcs.core.public_api import public_names_from_objects +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, @@ -123,7 +124,7 @@ class ExcessObjectHandling(CellProfilerEnumAttributeMixin, Enum): @numpy(contract=ProcessingContract.PURE_2D) def identify_primary_objects( - image: np.ndarray, + image: RuntimeArrayData, min_diameter: int = 10, max_diameter: int = 40, exclude_size: bool = True, diff --git a/openhcs/processing/backends/cellprofiler/relationships.py b/openhcs/processing/backends/cellprofiler/relationships.py index d5f1be0393..ced3fe559d 100644 --- a/openhcs/processing/backends/cellprofiler/relationships.py +++ b/openhcs/processing/backends/cellprofiler/relationships.py @@ -5,6 +5,9 @@ from dataclasses import dataclass, replace from enum import Enum from typing import TYPE_CHECKING, Annotated +from openhcs.core.runtime_relationships import ( + DirectParentReferenceFeatureMarker, +) from openhcs.core.alias_property import AliasProperty from openhcs.core.artifacts import ( ArtifactInputPlan, @@ -72,7 +75,6 @@ ObjectLocationMeasurementRows, ) from openhcs.interop.cellprofiler.runtime.relationship_measurement_rows import ( - DirectParentReferenceFeatureMarker, RelationshipMeasurementRows, ) @@ -862,6 +864,7 @@ def aggregate_child_feature_name( ) from openhcs.core.runtime_tabular_values import ColumnarRows from openhcs.core.runtime_object_labels import ( + dense_label_centers_2d_numba, ObjectLabelValue, object_label_value_with_dense_labels, ) @@ -1851,7 +1854,7 @@ def label_centers(self, labels: np.ndarray) -> np.ndarray: label_count = int(labels.max()) if label_count == 0: return np.empty((0, 2), dtype=np.float64) - centroids = _label_centroids_numba(np.ascontiguousarray(labels), label_count) + centroids = dense_label_centers_2d_numba(np.ascontiguousarray(labels), label_count) return centroids[1:] @@ -2190,29 +2193,6 @@ def _parents_of_from_overlap_counts_numba( return parents_of -@njit(cache=True) -def _label_centroids_numba(labels: np.ndarray, label_count: int) -> np.ndarray: - sums = np.zeros((label_count + 1, 2), dtype=np.float64) - counts = np.zeros(label_count + 1, dtype=np.int64) - height, width = labels.shape - for row in range(height): - for col in range(width): - label_id = int(labels[row, col]) - if label_id > 0 and label_id <= label_count: - sums[label_id, 0] += row - sums[label_id, 1] += col - counts[label_id] += 1 - centroids = np.empty((label_count + 1, 2), dtype=np.float64) - for label_id in range(label_count + 1): - if counts[label_id] == 0: - centroids[label_id, 0] = np.nan - centroids[label_id, 1] = np.nan - else: - centroids[label_id, 0] = sums[label_id, 0] / counts[label_id] - centroids[label_id, 1] = sums[label_id, 1] / counts[label_id] - return centroids - - @njit(cache=True) def _calculate_centroid_distances_numba( parent_labels: np.ndarray, @@ -2226,8 +2206,8 @@ def _calculate_centroid_distances_numba( distances[child_idx] = np.nan if child_count == 0 or parent_count == 0: return distances - parent_centroids = _label_centroids_numba(parent_labels, parent_count) - child_centroids = _label_centroids_numba(child_labels, child_count) + parent_centroids = dense_label_centers_2d_numba(parent_labels, parent_count) + child_centroids = dense_label_centers_2d_numba(child_labels, child_count) for child_idx in range(child_count): parent_id = int(parents_of[child_idx]) child_id = child_idx + 1 @@ -2279,7 +2259,7 @@ def _calculate_minimum_distances_numba( distances[child_idx] = np.nan if child_count == 0 or parent_count == 0: return distances - child_centroids = _label_centroids_numba(child_labels, child_count) + child_centroids = dense_label_centers_2d_numba(child_labels, child_count) height, width = parent_labels.shape counts = np.zeros(parent_count + 1, dtype=np.int64) for row in range(height): diff --git a/openhcs/processing/backends/cellprofiler/secondary.py b/openhcs/processing/backends/cellprofiler/secondary.py index 973a360374..d31dbc0f8c 100644 --- a/openhcs/processing/backends/cellprofiler/secondary.py +++ b/openhcs/processing/backends/cellprofiler/secondary.py @@ -38,6 +38,7 @@ EnumKeyedStrategyMixin, RegisteredLeafClassSpec, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_mask, @@ -1147,7 +1148,7 @@ def _replacement_primary_output_from_relationship( def _execute_identify_secondary_objects( - image: np.ndarray, + image: RuntimeArrayData, primary_labels: ObjectLabelValue, method: SecondaryMethod = SecondaryMethod.PROPAGATION, threshold_scope: CellProfilerThresholdScope = CellProfilerThresholdScope.GLOBAL, @@ -1337,7 +1338,7 @@ def _execute_identify_secondary_objects( @numpy(contract=ProcessingContract.PURE_2D) @special_inputs("primary_labels") def identify_secondary_objects( - image: np.ndarray, + image: RuntimeArrayData, primary_labels: ObjectLabelValue, method: SecondaryMethod = SecondaryMethod.PROPAGATION, threshold_scope: CellProfilerThresholdScope = CellProfilerThresholdScope.GLOBAL, @@ -1407,7 +1408,7 @@ def identify_secondary_objects( @numpy(contract=ProcessingContract.PURE_2D) @special_inputs("primary_labels") def identify_secondary_objects_with_replacement_primary( - image: np.ndarray, + image: RuntimeArrayData, primary_labels: ObjectLabelValue, method: SecondaryMethod = SecondaryMethod.PROPAGATION, threshold_scope: CellProfilerThresholdScope = CellProfilerThresholdScope.GLOBAL, diff --git a/openhcs/processing/backends/cellprofiler/skeleton.py b/openhcs/processing/backends/cellprofiler/skeleton.py index 947ca74c01..ba8c710ced 100644 --- a/openhcs/processing/backends/cellprofiler/skeleton.py +++ b/openhcs/processing/backends/cellprofiler/skeleton.py @@ -359,7 +359,7 @@ def measure_object_skeleton( @numpy_backend(contract=ProcessingContract.PURE_2D) @special_inputs("seed_labels") def measure_object_skeleton_with_branchpoint_image( - image: np.ndarray, + image: RuntimeArrayData, seed_labels: SeedObjectLabelsInput, fill_small_holes: bool = True, maximum_hole_size: int = 10, diff --git a/openhcs/processing/backends/cellprofiler/smoothing.py b/openhcs/processing/backends/cellprofiler/smoothing.py index 0b803b2294..6ac97257a9 100644 --- a/openhcs/processing/backends/cellprofiler/smoothing.py +++ b/openhcs/processing/backends/cellprofiler/smoothing.py @@ -5,6 +5,7 @@ from typing import Any from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode from openhcs.core.callable_contract import runtime_image_execution_mode +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, with_image_payload_data, @@ -790,7 +791,7 @@ def _fit_polynomial( def smooth_image( - image: np.ndarray, + image: RuntimeArrayData, smoothing_method: SmoothingMethod = SmoothingMethod.GAUSSIAN_FILTER, auto_object_size: bool = True, object_size: float = 16.0, @@ -839,7 +840,7 @@ def smooth_image( @numpy_decorator(contract=ProcessingContract.PURE_2D) def smooth( - image: np.ndarray, + image: RuntimeArrayData, smoothing_method: SmoothingMethod = SmoothingMethod.GAUSSIAN_FILTER, auto_object_size: bool = True, object_size: float = 16.0, @@ -862,7 +863,7 @@ def smooth( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.FLEXIBLE) def reducenoise( - image: np.ndarray, + image: RuntimeArrayData, patch_size: int = 5, patch_distance: int = 6, cutoff_distance: float = 0.1, diff --git a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py index 3682699839..3c0745c822 100644 --- a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py +++ b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py @@ -58,7 +58,7 @@ source_component_metadata_consensus, ) from openhcs.core.source_metadata import ( - SourceMetadataRoleView, + SourceMetadataFields, ) from openhcs.interop.cellprofiler.image_set_numbering import ( CellProfilerImageSetNumbering, @@ -583,8 +583,7 @@ def _source_metadata_measurement_rows( ).source_component_metadata if metadata is None: continue - role_view = SourceMetadataRoleView(metadata) - original_metadata = dict(role_view.original_items()) + original_metadata = dict(SourceMetadataFields.original_items(metadata)) if original_metadata: rows.append((image_numbers_by_slice[slice_index], original_metadata)) return tuple(rows) diff --git a/openhcs/processing/backends/cellprofiler/thresholding.py b/openhcs/processing/backends/cellprofiler/thresholding.py index eeac9dd3fb..7cd50a34d1 100644 --- a/openhcs/processing/backends/cellprofiler/thresholding.py +++ b/openhcs/processing/backends/cellprofiler/thresholding.py @@ -45,6 +45,7 @@ from openhcs.core.pipeline.function_contracts import runtime_bound_parameters from openhcs.core.public_api import public_names_from_objects from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_intensity_scale_for_dtype, @@ -549,7 +550,7 @@ class GlobalThresholdSourceSelection: method_parameters: GlobalThresholdMethodParameters -def normalize_cellprofiler_image(image: np.ndarray) -> np.ndarray: +def normalize_cellprofiler_image(image: RuntimeArrayData) -> np.ndarray: """Return an image in CellProfiler's normalized pixel-data convention.""" return image_payload_data( normalize_cellprofiler_image_payload( @@ -2578,7 +2579,7 @@ class _ThresholdEmbeddedMaskRuntimeParameter(KeywordRuntimeParameter): @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy(contract=ProcessingContract.PURE_2D) def threshold( - image: np.ndarray, + image: RuntimeArrayData, mask: np.ndarray | None = None, threshold_scope: ThresholdScope = ThresholdScope.GLOBAL, threshold_method: ThresholdMethod = ThresholdMethod.OTSU, diff --git a/openhcs/processing/backends/cellprofiler/tracking.py b/openhcs/processing/backends/cellprofiler/tracking.py index 3a36e51e4e..e591ba0d42 100644 --- a/openhcs/processing/backends/cellprofiler/tracking.py +++ b/openhcs/processing/backends/cellprofiler/tracking.py @@ -39,6 +39,7 @@ from openhcs.core.registry_strategies import EnumKeyedStrategyMixin from openhcs.core.runtime_object_labels import ( ObjectLabelValue, + dense_label_centers_2d_numba, object_label_dense_array, ) from openhcs.core.runtime_relationships import ObjectRelationshipDeclaration @@ -830,7 +831,7 @@ def label_centers(self, labels: np.ndarray) -> tuple[np.ndarray, np.ndarray]: label_count = int(labels_array.max()) if labels_array.size else 0 if label_count == 0: return (np.array([], dtype=np.float64), np.array([], dtype=np.float64)) - centers = _label_centers_numba(np.ascontiguousarray(labels_array), label_count) + centers = dense_label_centers_2d_numba(np.ascontiguousarray(labels_array), label_count) return (centers[1:, 0], centers[1:, 1]) def track_by_overlap( @@ -1461,29 +1462,6 @@ def _apply_final_age_measurements(frame_results: TrackingFrameResults) -> None: final_age_records[slice_index, object_label].final_age = int(lifetime) -@njit(cache=True) -def _label_centers_numba(labels: np.ndarray, label_count: int) -> np.ndarray: - sums = np.zeros((label_count + 1, 2), dtype=np.float64) - counts = np.zeros(label_count + 1, dtype=np.int64) - height, width = labels.shape - for y in range(height): - for x in range(width): - label_id = int(labels[y, x]) - if label_id > 0 and label_id <= label_count: - sums[label_id, 0] += y - sums[label_id, 1] += x - counts[label_id] += 1 - centers = np.empty((label_count + 1, 2), dtype=np.float64) - for label_id in range(label_count + 1): - if counts[label_id] == 0: - centers[label_id, 0] = np.nan - centers[label_id, 1] = np.nan - else: - centers[label_id, 0] = sums[label_id, 0] / counts[label_id] - centers[label_id, 1] = sums[label_id, 1] / counts[label_id] - return centers - - @njit(cache=True) def _track_by_overlap_numba( frame: TrackingKernelFrame, @@ -1528,8 +1506,8 @@ def _track_by_overlap_numba( def _track_by_distance_numba( frame: TrackingKernelFrame, pixel_radius: int ) -> tuple[np.ndarray, np.ndarray, np.ndarray, int]: - current_centers = _label_centers_numba(frame.current_labels, frame.current_count) - old_centers = _label_centers_numba(frame.old_labels, frame.old_count) + current_centers = dense_label_centers_2d_numba(frame.current_labels, frame.current_count) + old_centers = dense_label_centers_2d_numba(frame.old_labels, frame.old_count) new_labels = np.zeros(frame.current_count, dtype=np.int64) parent_object_numbers = np.zeros(frame.current_count, dtype=np.int64) parent_image_numbers = np.zeros(frame.current_count, dtype=np.int64) diff --git a/openhcs/processing/backends/cellprofiler/worms.py b/openhcs/processing/backends/cellprofiler/worms.py index 240070daef..1f0964c1a6 100644 --- a/openhcs/processing/backends/cellprofiler/worms.py +++ b/openhcs/processing/backends/cellprofiler/worms.py @@ -1709,7 +1709,7 @@ def canonical_image_output( def _execute_untangle_worms( - image: np.ndarray, + image: RuntimeArrayData, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, num_control_points: int = 21, @@ -1868,7 +1868,7 @@ def _execute_untangle_worms( def _untangle_worms_output( - image: np.ndarray, + image: RuntimeArrayData, *, expected_style: OverlapStyle, overlap_style: OverlapStyle, @@ -1940,7 +1940,7 @@ def _untangle_worms_output( @numpy(contract=ProcessingContract.PURE_2D) def untangle_worms( - image: np.ndarray, + image: RuntimeArrayData, overlap_style: OverlapStyle = OverlapStyle.WITHOUT_OVERLAP, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, @@ -1993,7 +1993,7 @@ def untangle_worms( @numpy(contract=ProcessingContract.PURE_2D) def untangle_worms_with_overlap( - image: np.ndarray, + image: RuntimeArrayData, overlap_style: OverlapStyle = OverlapStyle.WITH_OVERLAP, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, @@ -2046,7 +2046,7 @@ def untangle_worms_with_overlap( @numpy(contract=ProcessingContract.PURE_2D) def untangle_worms_both( - image: np.ndarray, + image: RuntimeArrayData, overlap_style: OverlapStyle = OverlapStyle.BOTH, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, @@ -2112,7 +2112,7 @@ def untangle_worms_both( @special_inputs("worm_labels") @runtime_bound_parameters(_StraightenWormControlPointsRuntimeParameter) def straighten_worms( - image: np.ndarray, + image: RuntimeArrayData, worm_labels: ObjectLabelValue, control_points: np.ndarray | None = None, worm_width: int = 20, @@ -2226,7 +2226,7 @@ def straighten_worms( @numpy(contract=ProcessingContract.PURE_2D) def identify_dead_worms( - image: np.ndarray, + image: RuntimeArrayData, worm_width: int = 10, worm_length: int = 100, angle_count: int = 32, diff --git a/openhcs/processing/backends/lib_registry/registry_service.py b/openhcs/processing/backends/lib_registry/registry_service.py index 453e1f734a..dd327c12a5 100644 --- a/openhcs/processing/backends/lib_registry/registry_service.py +++ b/openhcs/processing/backends/lib_registry/registry_service.py @@ -148,11 +148,11 @@ def prepare_in_current_process( cls._metadata_cache = cls._metadata_from_instances( cls._available_registry_instances() ) - callables = tuple( + callables = tuple(dict.fromkeys( target for metadata in cls._metadata_cache.values() for target in CallableProjection.from_callable(metadata.func).prepare_targets() - ) + )) emit_status("Preparing registered kernel caches") PreparationCacheBatch.from_callables(callables).populate_child_caches( max_workers=os.cpu_count() or 1, @@ -164,6 +164,9 @@ def prepare_in_current_process( emit_status( f"Prepared callable {preparation.projection.module_name}.{preparation.projection.name}" ) + emit_status("Resolving prepared callable signatures") + for func in callables: + CallableProjection.from_callable(func).warm_canonical_signature() emit_status("Preparing registered configuration source declarations") for declaration in dict.fromkeys( declaration diff --git a/openhcs/processing/backends/lib_registry/unified_registry.py b/openhcs/processing/backends/lib_registry/unified_registry.py index 4e60c05d3d..a30e9af840 100644 --- a/openhcs/processing/backends/lib_registry/unified_registry.py +++ b/openhcs/processing/backends/lib_registry/unified_registry.py @@ -75,7 +75,6 @@ from openhcs.core.runtime_array_values import RuntimeArrayPayload, is_array_payload from openhcs.core.runtime_batch_contracts import ( Pure2DSliceBatchExecutor, - RuntimeBatchExecutionDomain, RuntimePure2DSliceBatchRequest, runtime_batch_executors_from_callable, ) @@ -207,6 +206,7 @@ def accepted_kwargs( self, func: Callable[..., Any], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> dict[str, Any]: """Return kwargs accepted by ``func`` under this policy.""" @@ -220,8 +220,9 @@ def accepted_kwargs( self, func: Callable[..., Any], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> dict[str, Any]: - del func + del func, signature return dict(kwargs) @@ -234,8 +235,11 @@ def accepted_kwargs( self, func: Callable[..., Any], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> dict[str, Any]: - parameters = _runtime_callable_parameters(func) + from openhcs.core.callable_contract import CallableMetadata + + parameters = (CallableMetadata.callable_signature(func) if signature is None else signature).parameters if any( parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() @@ -255,6 +259,7 @@ class RuntimeCallableInvocation: kwarg_policy: RuntimeInvocationKwargPolicy = ( RuntimeInvocationKwargPolicy.PASS_THROUGH ) + signature: inspect.Signature | None = None def call(self) -> Any: target = RuntimeCallableViewStrategy.for_view(self.callable_view).resolve( @@ -264,7 +269,7 @@ def call(self) -> Any: *self.args, **RuntimeInvocationKwargPolicyStrategy.for_policy( self.kwarg_policy - ).accepted_kwargs(target, self.kwargs), + ).accepted_kwargs(target, self.kwargs, signature=self.signature), ) @@ -277,11 +282,22 @@ class RuntimeCallablePolicy: RuntimeInvocationKwargPolicy.PASS_THROUGH ) + def contract_invocation( + self, contract: "CallableContract", func: Callable[..., Any], + image: Any, kwargs: Mapping[str, Any], + ) -> RuntimeCallableInvocation: + """Bind the prepared main-image carrier and actual raw execution ABI.""" + return self.invocation( + func, (contract.raw_main_flow_call_argument(image),), kwargs, + signature=contract.raw_runtime_signature, + ) + def invocation( self, func: Callable[..., Any], args: tuple[Any, ...], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> RuntimeCallableInvocation: return RuntimeCallableInvocation( func=func, @@ -289,16 +305,10 @@ def invocation( kwargs=kwargs, callable_view=self.callable_view, kwarg_policy=self.kwarg_policy, + signature=signature, ) -@lru_cache(maxsize=256) -def _runtime_callable_parameters( - func: Callable[..., Any], -) -> Mapping[str, inspect.Parameter]: - return inspect.signature(func).parameters - - def _registry_runtime_parameter_exclusions( signature: inspect.Signature, parameter_names: tuple[str, ...], @@ -1012,7 +1022,7 @@ def main_flow_call_argument( self, callable_contract: "CallableContract", source_payload: Any, ) -> Any: """Expose the raw ABI when this contract needs no earlier plane slicing.""" - return callable_contract.raw_main_flow_call_argument(source_payload) + return callable_contract.runtime_main_flow_call_argument(source_payload) def injected_semantic_control_parameter_names(self) -> frozenset[str]: """Semantic controls that this contract may inject into public callables.""" @@ -1027,10 +1037,14 @@ def consume_semantic_controls( kwargs: MutableMapping[str, Any], *, func: Callable[..., Any] | None = None, + parameters: Mapping[str, inspect.Parameter] | None = None, ) -> dict[str, Any]: """Consume and return semantic selectors owned by this declaration.""" - parameters = {} if func is None else inspect.signature(func).parameters + if parameters is None: + from openhcs.core.callable_contract import CallableMetadata + + parameters = {} if func is None else CallableMetadata.callable_signature(func).parameters values: dict[str, Any] = {} for parameter_type in self.runtime_parameter_types(): if not parameter_type.is_semantic_control: @@ -1739,7 +1753,7 @@ def execute_pure_3d(self, func, image, *args, **kwargs): RuntimeCallablePolicy() .invocation( func, - (callable_contract.raw_main_flow_call_argument(image), *args), + (callable_contract.runtime_main_flow_call_argument(image), *args), kwargs, ) .call() @@ -1773,7 +1787,7 @@ def execute_pure_2d(self, func, image, *args, **kwargs): result = ( RuntimeCallablePolicy().invocation( func, - (callable_contract.raw_main_flow_call_argument(image), *args), + (callable_contract.runtime_main_flow_call_argument(image), *args), kwargs, ).call() ) @@ -1808,14 +1822,7 @@ def execute_pure_2d(self, func, image, *args, **kwargs): kwargs.setdefault(parameter.name, value) args = args[len(positional_parameters) :] slices = slicer.slice_value(image, input_memory_type) - declared_batch_executor = runtime_batch_executors_from_callable(func).get( - RuntimeBatchExecutionDomain.PURE_2D_SLICES - ) - batch_executor = ( - declared_batch_executor - if callable(declared_batch_executor) - else Pure2DSliceBatchExecutor.default_executor() - ) + batch_executor = Pure2DSliceBatchExecutor.from_executors(runtime_batch_executors_from_callable(func)) def execute_slice( slice_func: Callable[..., Any], @@ -1837,7 +1844,7 @@ def execute_slice( RuntimeCallablePolicy() .invocation( slice_func, - (callable_contract.raw_main_flow_call_argument(slice_2d), *args), + (callable_contract.runtime_main_flow_call_argument(slice_2d), *args), projected_kwargs, ) .call() diff --git a/openhcs/processing/materialization/__init__.py b/openhcs/processing/materialization/__init__.py index 4e920f0ee9..c161998f13 100644 --- a/openhcs/processing/materialization/__init__.py +++ b/openhcs/processing/materialization/__init__.py @@ -6,11 +6,14 @@ ) from openhcs.processing.materialization.core import ( BackendSaver, + MaterializationBatch, MaterializationContext, MaterializationSpec, Output, + SavedMaterializationOutputs, PathHelper, materialize, + prepare_materialization, materialization_outputs, registered_materialization_option_types, tabular_field_names_from_materialization, @@ -57,11 +60,14 @@ "MaterializationSpec", "TerminalMaterializationSpec", "StreamingOnlyMaterializationSpec", + "MaterializationBatch", "MaterializationContext", "Output", + "SavedMaterializationOutputs", "PathHelper", "BackendSaver", "materialize", + "prepare_materialization", "materialization_outputs", "registered_materialization_option_types", "FileOutputOptions", diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index faff29684a..74efd7882d 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -17,6 +17,7 @@ from dataclasses import dataclass, field, is_dataclass, replace from functools import lru_cache, singledispatch from pathlib import Path, PurePosixPath, PureWindowsPath +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, TypeAlias import numpy as np @@ -72,11 +73,16 @@ source_component_metadata_value, source_metadata_value, ) +from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.source_spatial_domain import SourceSpatialDomain from openhcs.core.steps.stream_component_semantics import ( StreamImagePayloadMetadataProjector, StreamViewerComponentMetadataProjector, ) +from openhcs.core.steps.function_output_identity import ( + FunctionOutputIdentityAuthority, + FunctionOutputPathAuthority, +) from openhcs.processing.materialization.constants import ( MaterializationFormat, WriteMode, @@ -344,6 +350,17 @@ def with_variable_components( ) +@dataclass(frozen=True, kw_only=True) +class PointROIOutput(Output): + """Full point-archive provenance with an exact source-plane stream anchor.""" + + metadata: ImagePayloadMetadata = field() + + @property + def source_identity(self) -> SourceImageIdentity: + return self.metadata.source_provenance.for_source_plane(0).scalar_source_identity + + @dataclass(frozen=True) class TextOutput(Output): """Materialization output whose canonical payload is text.""" @@ -1651,7 +1668,8 @@ def __init__( def save_all( self, outputs: Sequence[Output], - ) -> None: + ) -> Mapping[str, tuple[Output, ...]]: + saved_outputs: dict[str, tuple[Output, ...]] = {} for backend in self.backends: backend_instance = self.filemanager._get_backend(backend) output_acceptance = tuple( @@ -1690,6 +1708,8 @@ def save_all( backend, **kwargs, ) + saved_outputs[backend] = supported_outputs + return saved_outputs def _prepare_path(self, backend: str, backend_instance, path: str) -> None: if not backend_instance.requires_filesystem_validation: @@ -1724,6 +1744,25 @@ class MaterializationContext: source_paths: tuple[str, ...] = () pipeline_position: int | None = None output_plan: ArtifactOutputPlan | None = None + materialization_spec: MaterializationSpec | None = None + + def named_source_filename( + self, metadata: ImagePayloadMetadata, extension: str, + ) -> str | None: + """Name a retained image from the actual rendering purpose and role.""" + if self.materialization_spec is None: + return None + qualifier = self.materialization_spec.filename_qualifier(self.output_plan) + if qualifier is None: + return None + parser = SourceStemAuthoritySelection.from_processing_context(self.context).required_parser() + identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata(parser, metadata) + if identity is None: + raise ValueError("Retained image output has no addressable source filename identity.") + identity = self.materialization_spec.filename_identity_for_output( + replace(identity, extension=extension), self.output_plan, + ) + return FunctionOutputPathAuthority.filename_for_identity(parser, identity) def paths(self, options: FileOutputOptions) -> PathHelper: return PathHelper(self.base_path, options) @@ -2503,32 +2542,23 @@ def write_image_file( options.filename_identity is MaterializedFilenameIdentity.ARTIFACT_NAME or paths.source_identity_base_path(options) is not None ) - outputs = tuple( - ( - ( - _image_relative_output_path( - item.value, - options, - context, - sequence_index=sequence_index, - ) - if options.relative_path_template is not None - else ( - paths.primary_output_path(options) - if preserves_planned_path - else str( - paths.parent - / ( - source_stem_authority.required_source_stem(item.metadata) - + options.primary_output_suffix - ) - ) + outputs = [] + for sequence_index, item in enumerate(projected_items, start=1): + if options.relative_path_template is not None: + path = _image_relative_output_path( + item.value, options, context, sequence_index=sequence_index, + ) + elif preserves_planned_path: + path = paths.primary_output_path(options) + else: + filename = context.named_source_filename(item.metadata, options.primary_output_suffix) + if filename is None: + filename = ( + source_stem_authority.required_source_stem(item.metadata) + + options.primary_output_suffix ) - ), - item, - ) - for sequence_index, item in enumerate(projected_items, start=1) - ) + path = str(paths.parent / filename) + outputs.append((path, item)) output_paths = tuple(path for path, _item in outputs) if len(set(output_paths)) != len(output_paths): raise ValueError( @@ -3184,11 +3214,8 @@ def _write_point_roi_zip( from polystore.roi import PointShape, ROI from openhcs.core.roi_point_metadata import ROIFractionalZ from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata - from openhcs.core.runtime_measurements import MeasurementTable - payload = MaterializationInput.from_value(data, options).data - if not isinstance(payload, MeasurementTable): - raise TypeError("PointROIOptions requires a MeasurementTable payload.") + payload = options.measurement_payload(data) id_field = payload.subject.object_id_field if id_field is None: raise ValueError("PointROIOptions requires an object-measurement subject.") @@ -3259,7 +3286,7 @@ def _write_point_roi_zip( metadata = ImagePayloadMetadata(source_provenance=provenance) ROIFractionalZ.source_component_domain(rois, metadata) return [ - Output( + PointROIOutput( path=ctx.paths(options).primary_output_path(options), content=ROIArchiveSourceMetadata.bind(rois, metadata), metadata=metadata, @@ -3280,6 +3307,8 @@ def _write_spatial_graph_roi_zip( from polystore.roi import ROI, PolylineShape + from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata + materialization_input = MaterializationInput.from_value(data, options) graph = materialization_input.data if not isinstance(graph, SpatialGraph): @@ -3288,6 +3317,10 @@ def _write_spatial_graph_roi_zip( f"got {type(graph).__name__}." ) + source_metadata = ImagePayloadMetadata( + source_provenance=graph.source_provenance, + source_voxel_spacing=SourceVoxelSpacing(graph.coordinate_spacing), + ) rois = [] for edge in graph.edges: coordinates = np.asarray(edge.coordinates, dtype=float) @@ -3329,7 +3362,8 @@ def _write_spatial_graph_roi_zip( return [ Output( path=ctx.paths(options).primary_output_path(options), - content=rois, + content=ROIArchiveSourceMetadata.bind(rois, source_metadata), + metadata=source_metadata, ) ] @@ -3995,6 +4029,10 @@ def participates_in_runtime_export_observation(self) -> bool: """Explicit materialization specs are externally observed exports.""" return True + def filename_qualifier(self, output_plan: ArtifactOutputPlan | None) -> str | None: + """Preserve the source filenames authored by an explicit export.""" + return None + def participates_in_persistent_materialization(self) -> bool: """Explicit materialization specs write to configured persistent targets.""" return True @@ -4028,34 +4066,15 @@ def emits_variable_component_planes(self, data: MaterializationValue) -> bool: for output_options in self.outputs ) - def emitted_source_identities( + def stream_source_identities( self, data: MaterializationValue, ) -> tuple[SourceImageIdentity, ...]: - """Return source identities for writers that emit projected planes.""" + """Return the source domain represented by the selected writer stream.""" for output_options in self.outputs: - if not materialization_emits_variable_component_planes( - output_options, - data, - ): - continue - materialization_input = ( - MaterializationInput.from_runtime_slice_projected_value( - data, - output_options, - ) - ) - if len(materialization_input.items) == 1: - provenance = materialization_input.items[0].metadata.source_provenance - if provenance.source_plane_count > 1: - return tuple( - provenance.for_source_plane(index).scalar_source_identity - for index in range(provenance.source_plane_count) - ) - return tuple( - item.metadata.source_provenance.scalar_source_identity - for item in materialization_input.items - ) + identities = output_options.stream_source_identities(data) + if identities: + return identities return () def uses_filename_source_identity(self, data: MaterializationValue) -> bool: @@ -4129,28 +4148,70 @@ def validate(spec: MaterializationSpec, backends: list[str]) -> None: ) -def _materialization_output_groups( - spec: MaterializationSpec, - data: MaterializationValue, - context: MaterializationContext, - *, - output_path_filter: Callable[[Path], bool] | None = None, -) -> tuple[tuple[WriterSpec, tuple[Output, ...]], ...]: - """Render requested outputs through each writer's declared path projection.""" +@dataclass(frozen=True) +class SavedMaterializationOutputs: + """Backend-indexed outputs from completed saves, independent of artifact kind.""" - return tuple( - ( - writer, - tuple( - output.with_source_identity_fallback( - context.artifact_source_identity - ).with_variable_components(context.variable_components) - for output in writer.outputs(data, options, context, output_path_filter) - ), + outputs_by_backend: Mapping[str, tuple[Output, ...]] + + def outputs_for_backend(self, backend: str) -> tuple[Output, ...]: + return self.outputs_by_backend.get(backend, ()) + + +@dataclass(frozen=True) +class MaterializationBatch: + """One rendered writer batch shared by saving and immediate publication.""" + + context: MaterializationContext + output_groups: tuple[tuple[WriterSpec, tuple[Output, ...]], ...] + primary: int + + @classmethod + def render( + cls, + spec: MaterializationSpec, + data: MaterializationValue, + context: MaterializationContext, + *, + output_path_filter: Callable[[Path], bool] | None = None, + ) -> MaterializationBatch: + context = replace(context, materialization_spec=spec) + groups = tuple( + ( + writer, + tuple( + output.with_source_identity_fallback( + context.artifact_source_identity + ).with_variable_components(context.variable_components) + for output in writer.outputs( + data, options, context, output_path_filter + ) + ), + ) + for options in spec.outputs + for writer in (_WRITERS_BY_OPTIONS[options.__class__],) ) - for options in spec.outputs - for writer in (_WRITERS_BY_OPTIONS[options.__class__],) - ) + return cls(context=context, output_groups=groups, primary=spec.primary) + + @property + def outputs(self) -> tuple[Output, ...]: + return tuple( + output for _writer, outputs in self.output_groups for output in outputs + ) + + @property + def primary_path(self) -> str: + writer, outputs = self.output_groups[self.primary] + return writer.primary_path(list(outputs)) + + def save(self) -> SavedMaterializationOutputs: + """Return only outputs accepted and successfully saved by each backend.""" + saved: dict[str, tuple[Output, ...]] = {} + saver = self.context.saver + for _writer, outputs in self.output_groups: + for backend, saved_outputs in saver.save_all(outputs).items(): + saved[backend] = (*saved.get(backend, ()), *saved_outputs) + return SavedMaterializationOutputs(MappingProxyType(saved)) def materialization_outputs( @@ -4190,19 +4251,15 @@ def materialization_outputs( pipeline_position=pipeline_position, output_plan=output_plan, ) - return tuple( - output - for _writer, outputs in _materialization_output_groups( - spec, - data, - materialization_context, - output_path_filter=output_path_filter, - ) - for output in outputs - ) + return MaterializationBatch.render( + spec, + data, + materialization_context, + output_path_filter=output_path_filter, + ).outputs -def materialize( +def prepare_materialization( spec: MaterializationSpec, data: MaterializationValue, path: str, @@ -4218,8 +4275,8 @@ def materialize( source_paths: Sequence[str] = (), pipeline_position: int | None = None, output_plan: ArtifactOutputPlan | None = None, -) -> str: - """Materialize data to one or more backends.""" +) -> MaterializationBatch: + """Prepare one exact batch for saving and immediate publication.""" normalized_backends = BackendSequenceAuthority.normalize(backends) AllowedBackendsAuthority.validate(spec, normalized_backends) @@ -4245,11 +4302,42 @@ def materialize( output_plan=output_plan, ) - primary_path = "" + return MaterializationBatch.render(spec, data, ctx) - for i, (writer, outs) in enumerate(_materialization_output_groups(spec, data, ctx)): - ctx.saver.save_all(outs) - if i == spec.primary: - primary_path = writer.primary_path(list(outs)) - return primary_path +def materialize( + spec: MaterializationSpec, + data: MaterializationValue, + path: str, + filemanager: FileManager, + backends: Sequence[str] | str, + backend_kwargs: BackendKwargsInput = BACKEND_KWARGS_ABSENT, + context: ProcessingContext | None = None, + extra_inputs: dict | None = None, + *, + artifact_source_identity: SourceImageIdentity | None = None, + artifact_filename_identity: SourceImageIdentity | None = None, + variable_components: Sequence[VariableComponents] = (), + source_paths: Sequence[str] = (), + pipeline_position: int | None = None, + output_plan: ArtifactOutputPlan | None = None, +) -> str: + """Materialize data and return the primary path derived from its writer batch.""" + batch = prepare_materialization( + spec, + data, + path, + filemanager, + backends, + backend_kwargs, + context=context, + extra_inputs=extra_inputs, + artifact_source_identity=artifact_source_identity, + artifact_filename_identity=artifact_filename_identity, + variable_components=variable_components, + source_paths=source_paths, + pipeline_position=pipeline_position, + output_plan=output_plan, + ) + batch.save() + return batch.primary_path diff --git a/openhcs/processing/materialization/options.py b/openhcs/processing/materialization/options.py index d29b4e8f5f..8932702c60 100644 --- a/openhcs/processing/materialization/options.py +++ b/openhcs/processing/materialization/options.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional from openhcs.core.runtime_measurements import RuntimeMeasurementFeature from openhcs.processing.materialization.path_scopes import ( @@ -18,6 +18,11 @@ SharedMaterializationRelativePathScope, ) +if TYPE_CHECKING: + from openhcs.core.runtime_measurements import MeasurementTable + from openhcs.core.source_image_provenance import SourceImageIdentity + from openhcs.processing.materialization.core import MaterializationValue + class MaterializedFilenameIdentity(str, Enum): """Semantic identity used to construct materialized artifact filenames.""" @@ -53,6 +58,33 @@ def primary_output_suffix(self) -> str: """Suffix for this writer's primary materialized output.""" return self.filename_suffix + def stream_source_identities( + self, + data: MaterializationValue, + ) -> tuple[SourceImageIdentity, ...]: + """Return the source planes represented by this writer's stream.""" + from openhcs.processing.materialization.core import ( + MaterializationInput, + materialization_emits_variable_component_planes, + ) + + if not materialization_emits_variable_component_planes(self, data): + return () + materialization_input = MaterializationInput.from_runtime_slice_projected_value( + data, self + ) + if len(materialization_input.items) == 1: + provenance = materialization_input.items[0].metadata.source_provenance + if provenance.source_plane_count > 1: + return tuple( + provenance.for_source_plane(index).scalar_source_identity + for index in range(provenance.source_plane_count) + ) + return tuple( + item.metadata.source_provenance.scalar_source_identity + for item in materialization_input.items + ) + @dataclass(frozen=True) class SourceOptions: @@ -126,6 +158,27 @@ def __post_init__(self) -> None: if len({feature.measurement_row_field_name for feature in features}) != 3: raise ValueError("PointROIOptions coordinates require distinct row fields.") + def measurement_payload(self, data: MaterializationValue) -> MeasurementTable: + """Resolve the point writer's declared source to its typed measurements.""" + from openhcs.core.runtime_measurements import MeasurementTable + from openhcs.processing.materialization.core import MaterializationInput + + payload = MaterializationInput.from_value(data, self).data + if not isinstance(payload, MeasurementTable): + raise TypeError("PointROIOptions requires a MeasurementTable payload.") + return payload + + def stream_source_identities( + self, + data: MaterializationValue, + ) -> tuple[SourceImageIdentity, ...]: + """One point archive represents all original source planes, not one Z.""" + provenance = self.measurement_payload(data).source_provenance + return tuple( + provenance.for_source_plane(index).scalar_source_identity + for index in range(max(1, provenance.source_plane_count)) + ) + @dataclass(frozen=True) class SWCOptions(FileOutputOptions, SourceOptions): diff --git a/openhcs/processing/materialization/persistence.py b/openhcs/processing/materialization/persistence.py index 7d976d1d09..4fd675fc20 100644 --- a/openhcs/processing/materialization/persistence.py +++ b/openhcs/processing/materialization/persistence.py @@ -1,5 +1,11 @@ """Shared persistence and export-observation policies for materialized artifacts.""" +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openhcs.core.artifacts import ArtifactOutputPlan + from .core import MaterializationSpec @@ -9,6 +15,12 @@ class TerminalMaterializationSpec(MaterializationSpec): def participates_in_runtime_export_observation(self) -> bool: return False + def filename_qualifier(self, output_plan: ArtifactOutputPlan | None) -> str | None: + """Retain the compiled image role without changing its source address.""" + if output_plan is None: + return None + return output_plan.artifact_type.retained_filename_qualifier(output_plan.name) + class StreamingOnlyMaterializationSpec(MaterializationSpec): """Prepare viewer outputs without declaring persistent or external exports.""" diff --git a/openhcs/runtime/napari_streaming_handlers.py b/openhcs/runtime/napari_streaming_handlers.py index 43df62396e..4047ed9ff3 100644 --- a/openhcs/runtime/napari_streaming_handlers.py +++ b/openhcs/runtime/napari_streaming_handlers.py @@ -20,6 +20,7 @@ from openhcs.constants import AllComponents from openhcs.core.artifacts import ObjectArtifactSubjectBinding from openhcs.core.config import NapariDisplayConfig +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, ) @@ -1440,10 +1441,12 @@ def align_coordinates(self, coordinates: np.ndarray) -> np.ndarray: def translate( self, payload_axis_labels: tuple[str, ...] = (), + *, + scale: Sequence[float], ) -> tuple[float, ...]: - """Return translation in the same declaration-owned slots as layer data.""" + """Place route offsets in the same native world units as layer scale.""" - return ( + offsets = ( *( float(self.axis_offset(index)) for index in range(len(self.display_axis_components)) @@ -1452,16 +1455,19 @@ def translate( 0.0, 0.0, ) + return tuple( + offset * spacing for offset, spacing in zip(offsets, scale, strict=True) + ) def spatial_layer_kwargs( self, items: Sequence[NapariStreamLayerItem], payload_axis_labels: tuple[str, ...] = (), ) -> dict[str, LayerKwargValue]: - """Project calibrated XY onto aligned native axes for every layer kind. + """Project declared spatial calibration onto every aligned layer kind. - Component and internal payload axes remain dimensionless. Two-dimensional - acquisition calibration never invents physical Z or a color-band axis. + Only a real three-axis source declaration calibrates projected Z. + Acquisition selectors and internal payload bands remain dimensionless. """ spacing = CommonRuntimeValue.from_values( item.image_metadata.source_voxel_spacing for item in items @@ -1470,11 +1476,27 @@ def spatial_layer_kwargs( raise ValueError( "A native viewer route requires consistent source voxel spacing." ) - prefix = len(self.display_axis_components) + len(payload_axis_labels) + component_scales = [1.0] * len(self.display_axis_components) + component_units = ["dimensionless"] * len(self.display_axis_components) + z_component = AllComponents.Z_INDEX.value + if ( + len(spacing.values_zyx) == 3 + and z_component in self.display_axis_components + ): + z_axis = self.display_axis_components.index(z_component) + component_scales[z_axis] = spacing.spacing_for_ndim(3)[0] + component_units[z_axis] = spacing.native_coordinate_unit + scale = ( + *component_scales, + *(1.0 for _ in payload_axis_labels), + *spacing.spacing_for_ndim(2), + ) return { - "scale": (*(1.0 for _ in range(prefix)), *spacing.spacing_for_ndim(2)), + "scale": scale, + "translate": self.translate(payload_axis_labels, scale=scale), "units": ( - *("dimensionless" for _ in range(prefix)), + *component_units, + *("dimensionless" for _ in payload_axis_labels), *(spacing.native_coordinate_unit for _ in range(2)), ), } @@ -1901,7 +1923,7 @@ def append( for column_values in self.values.values(): column_values.append(None) - for name, value in metadata.items(): + for name, value in ROIArchiveSourceMetadata.feature_metadata(metadata).items(): if name in ( ObjectArtifactSubjectBinding.SUBJECT_FEATURE, ObjectArtifactSubjectBinding.SUBJECT_ID_FEATURE, diff --git a/openhcs/runtime/napari_viewer_server.py b/openhcs/runtime/napari_viewer_server.py index e16cb3e6b0..ef58616bd1 100644 --- a/openhcs/runtime/napari_viewer_server.py +++ b/openhcs/runtime/napari_viewer_server.py @@ -1164,13 +1164,14 @@ def _build_nd_points( all_properties["component"].append( metadata.value(VisualMetadataField.COMPONENT, 0) ) - point_metadata.append(metadata.metadata) + point_metadata.append( + ROIArchiveSourceMetadata.feature_metadata(metadata.metadata) + ) excluded = { "label", "component", ROIFractionalZ.FIELD, - ROIArchiveSourceMetadata.FIELD, } feature_fields = { key @@ -1840,14 +1841,12 @@ def handle(self, request: NapariLayerDisplayRequest) -> None: presentation, payload_axis_labels=payload_axis_labels, ) - translate = presentation.translate(payload_axis_labels) request = replace(request, presentation=presentation) axis_labels = presentation.axis_labels layer_kwargs = image_presentation.layer_kwargs(request.display_config.colormap) if axis_labels is not None: layer_kwargs["axis_labels"] = axis_labels - layer_kwargs["translate"] = translate layer_kwargs.update( presentation.spatial_layer_kwargs(layer_items, payload_axis_labels) ) @@ -2026,7 +2025,6 @@ def display_work( "face_color_cycle": color_projection.cycle, "opacity": 0.7, "ndim": shape_payload.ndim, - "translate": presentation.translate(), "visible": False, } if shape_payload.result_metadata: @@ -2062,7 +2060,7 @@ def geometric_component_values( items: Sequence[NapariStreamLayerItem], component_axis_semantics: ViewerComponentAxisSemantics, ) -> ComponentValues: - """Keep the declared Z span containing fractional-Z points navigable.""" + """Keep the represented source span, not just occupied point Z, navigable.""" z_component = AllComponents.Z_INDEX.value coordinates = tuple( (item, fractional_z) @@ -2083,7 +2081,7 @@ def geometric_component_values( domain = component_axis_semantics.required_component_values((z_component,))[ z_component ] - occupied: set[ComponentValue] = set() + represented: set[ComponentValue] = set() for item, fractional_z in coordinates: anchor = ViewerComponentCoordinateAuthority.required_value( item.address.components, @@ -2103,8 +2101,8 @@ def geometric_component_values( f"Fractional-Z point {fractional_z.value!r} exceeds the " f"declared Z domain from anchor {anchor!r}." ) - occupied.update(domain[start : stop + 1]) - return {z_component: [value for value in domain if value in occupied]} + represented.update(domain[start:]) + return {z_component: [value for value in domain if value in represented]} def handle(self, request: NapariLayerDisplayRequest) -> None: pipeline = request.pipeline @@ -2126,7 +2124,6 @@ def handle(self, request: NapariLayerDisplayRequest) -> None: "properties": properties, "face_color": "green", "size": 3, - "translate": presentation.translate(), } layer_kwargs.update(presentation.spatial_layer_kwargs(request.items)) if axis_labels is not None: @@ -2267,7 +2264,9 @@ def reconcile_mounted_axis_projections( if not apply: continue layer = self.server.layer_route_state.layer(route_key) - layer.translate = presentation.translate(presentation.payload_axis_labels) + layer.translate = presentation.spatial_layer_kwargs( + items, presentation.payload_axis_labels + )["translate"] self.dimension_label_store.apply(presentation) def schedule_layer_update( diff --git a/openhcs/runtime/zmq_execution_observation.py b/openhcs/runtime/zmq_execution_observation.py index fecffce412..f75dc3072d 100644 --- a/openhcs/runtime/zmq_execution_observation.py +++ b/openhcs/runtime/zmq_execution_observation.py @@ -9,7 +9,11 @@ from pathlib import Path from openhcs.core.context.processing_context import ProcessingContext -from openhcs.core.orchestrator.execution_result import ExecutionResult, ExecutionStatus +from openhcs.core.orchestrator.execution_result import ( + ExecutionResult, + RuntimeExecutionObservation, + ExecutionStatus, +) from openhcs.core.runtime_execution_validation import ( RuntimeArtifactExecutionExpectation, RuntimeArtifactExecutionObservation, @@ -215,11 +219,12 @@ def from_execution( compiled_contexts: Mapping[str, ProcessingContext], execution_results: Mapping[str, ExecutionResult], output_roots: tuple[Path, ...], + runtime_observations: tuple[RuntimeExecutionObservation, ...], server_environment: RuntimeEnvironmentSnapshot | None = None, execution_id: str | None = None, ) -> ZMQRuntimeExecutionObservationExport: observation = RuntimeArtifactExecutionObservation.from_contexts( - compiled_contexts + compiled_contexts, runtime_observations=runtime_observations ) return cls( schema_version=ZMQ_RUNTIME_OBSERVATION_EXPORT_SCHEMA_VERSION, diff --git a/openhcs/runtime/zmq_execution_server.py b/openhcs/runtime/zmq_execution_server.py index 5dc2a2c43d..e981bccf96 100644 --- a/openhcs/runtime/zmq_execution_server.py +++ b/openhcs/runtime/zmq_execution_server.py @@ -770,17 +770,8 @@ def _export_runtime_observation( is ZMQRuntimeObservationExportScope.OUTCOMES ): observed_exports = RuntimeExportObservation.from_runtime_observations( - tuple( - result.runtime_observation for result in execution_results.values() - ) + execution_results.runtime_observations ) - if execution_bundle.requires_parent_runtime_observation: - parent_exports = RuntimeExportObservation.from_execution_contexts( - execution_bundle.runtime_contexts - ) - observed_exports = RuntimeExportObservation.from_output_paths( - (*observed_exports.output_files, *parent_exports.output_files) - ) export = ZMQRuntimeExecutionOutcomeExport.from_execution( compiled_axis_ids=execution_bundle.runtime_contexts, execution_results=execution_results, @@ -793,6 +784,7 @@ def _export_runtime_observation( export = ZMQRuntimeExecutionObservationExport.from_execution( compiled_contexts=execution_bundle.runtime_contexts, execution_results=execution_results, + runtime_observations=execution_results.runtime_observations, output_roots=output_roots, server_environment=self._server_environment, execution_id=request_context.execution_id, diff --git a/tests/integration/test_source_store_code_mode_zmq.py b/tests/integration/test_source_store_code_mode_zmq.py index f8858301d9..749b273b4a 100644 --- a/tests/integration/test_source_store_code_mode_zmq.py +++ b/tests/integration/test_source_store_code_mode_zmq.py @@ -42,7 +42,7 @@ SourceFilterSubject, SourceSelector, ) -from openhcs.core.source_metadata import SourceMetadataRoleView +from openhcs.core.source_metadata import SourceMetadataFields from openhcs.core.steps.function_step import FunctionStep from openhcs.microscopes.bioformats_adapter import ( BioFormatsAdapterUnavailableError, @@ -316,9 +316,7 @@ def test_code_mode_and_zmq_wire_preserve_mixed_store_sources( } for alias, (source_path, pixels) in stores.items(): virtual_path, record = records_by_alias[alias] - filter_paths = SourceMetadataRoleView( - record.source_metadata - ).source_filter_paths() + filter_paths = SourceMetadataFields.source_filter_paths(record.source_metadata) assert source_path.name in filter_paths assert str(source_path) in filter_paths np.testing.assert_array_equal( diff --git a/tests/unit/agent/test_cold_inspection_feedback.py b/tests/unit/agent/test_cold_inspection_feedback.py index 57db9b0f62..32d6a7d586 100644 --- a/tests/unit/agent/test_cold_inspection_feedback.py +++ b/tests/unit/agent/test_cold_inspection_feedback.py @@ -63,7 +63,7 @@ def test_new_gateway_hook_cooperates_in_both_mros_before_terminal_result(tmp_pat """Independent audit capability needs no edits to service or MCP consumers.""" identity = ContextVar("cold-feedback-request", default="outside") calls, messages = [], [] - emitted = threading.Event() + compile_entered = threading.Event() release = threading.Event() error = ValueError("original declared compiler error") @@ -77,8 +77,8 @@ def compile(self, request): class DeclaredGateway(_FakeCompileInspectionGateway): def _compile(self, request): assert threading.current_thread() is threading.main_thread() + compile_entered.set() self.emit_progress(request) - emitted.set() assert release.wait(3), "Actual stage relay was not delivered during held work" if fails: raise error @@ -97,7 +97,7 @@ async def report_progress(self, progress, total=None, message=None): assert threading.current_thread() is not threading.main_thread() messages.append(message) if "compile: running: A01: compilation" in message: - assert emitted.is_set() + assert compile_entered.is_set() release.set() executor = McpTransportExecutor() @@ -105,7 +105,7 @@ async def report_progress(self, progress, total=None, message=None): try: for gateway_type in (Before, After): release.clear() - emitted.clear() + compile_entered.clear() gateway = gateway_type() service = ExecutionSessionService( path_policy=AgentPathPolicy.with_roots( diff --git a/tests/unit/test_aligned_main_flow_projection.py b/tests/unit/test_aligned_main_flow_projection.py index b4e4e05b03..0bc40ba4da 100644 --- a/tests/unit/test_aligned_main_flow_projection.py +++ b/tests/unit/test_aligned_main_flow_projection.py @@ -52,6 +52,62 @@ def _input_ref( return ArtifactSpec.input(name, artifact_type).ref() +@pytest.mark.parametrize("carrier_type", (AlignedImageStack, ImageOutputBundle)) +def test_aligned_stack_binds_complete_declared_output_roster( + carrier_type: type[AlignedImageStack], +) -> None: + first = ArtifactSpec.output("First", ImageArtifactType) + second = ArtifactSpec.output("Second", ImageArtifactType) + first_context, second_context = AlignedImageSliceContext.main_flow_for_artifact_specs( + (first, second) + ) + first_value = object() + carrier = carrier_type( + (None, first_value), + (second_context, first_context), + ) + + assert carrier.output_values_for_artifact_specs((first, second)) == { + first.ref(): first_value, + second.ref(): None, + } + with pytest.raises(ValueError, match="duplicate named contexts"): + carrier.output_values_for_artifact_specs((first, first)) + + +@pytest.mark.parametrize( + ("contexts", "message"), + ( + ((), "require exact AlignedImageStack"), + (("first", "first"), "contains duplicate context"), + (("first",), "does not carry every declared output"), + (("first", "unknown"), "context is not declared"), + (("first", "non-main"), "contains a non-main-flow"), + ), +) +def test_aligned_stack_rejects_incomplete_or_conflicting_output_rosters( + contexts: tuple[str, ...], + message: str, +) -> None: + first = ArtifactSpec.output("First", ImageArtifactType) + second = ArtifactSpec.output("Second", ImageArtifactType) + first_context, second_context = AlignedImageSliceContext.main_flow_for_artifact_specs( + (first, second) + ) + selected = { + "first": first_context, + "unknown": replace(second_context, output_key="Unknown"), + "non-main": replace(second_context, output_kind="artifact"), + } + carrier = AlignedImageStack( + tuple(object() for _ in contexts) if contexts else (object(), object()), + tuple(selected[name] for name in contexts), + ) + + with pytest.raises(ValueError, match=message): + carrier.output_values_for_artifact_specs((first, second)) + + def _named_carrier() -> tuple[AlignedImageStack, np.ndarray, np.ndarray]: mcherry = np.full((3, 4), 3.0, dtype=np.float32) gfp = np.full((3, 4), 2.0, dtype=np.float32) diff --git a/tests/unit/test_artifact_spec_collection.py b/tests/unit/test_artifact_spec_collection.py index db6e37dd1c..829bc2bcab 100644 --- a/tests/unit/test_artifact_spec_collection.py +++ b/tests/unit/test_artifact_spec_collection.py @@ -1,5 +1,7 @@ from __future__ import annotations +from dataclasses import replace + import pytest from openhcs.core.artifacts import ( @@ -9,6 +11,7 @@ ArtifactSpecCollection, GroupLineageSourceRelation, ImageArtifactType, + InputStackBroadcastSourceRelation, MainFlowPlaneProjectionOutputSpec, MeasurementsArtifactType, ObjectLabelsArtifactType, @@ -51,6 +54,45 @@ def test_artifact_spec_collection_queries_ordered_artifact_contracts() -> None: assert collection.by_name_and_artifact_type("Nuclei", ImageArtifactType) is None +def test_stack_broadcast_indices_use_exact_ordered_occurrences() -> None: + primary = ArtifactSpec.input("Primary", ImageArtifactType) + secondary = ArtifactSpec.input( + "Secondary", + ImageArtifactType, + relations=(InputStackBroadcastSourceRelation(source=primary.ref()),), + ) + unrelated = ArtifactSpec.input("Primary", ObjectLabelsArtifactType) + + assert ArtifactSpecCollection(()).stack_broadcast_source_indices() == () + assert ArtifactSpecCollection( + (secondary, unrelated, primary) + ).stack_broadcast_source_indices() == (2, None, None) + + for specs in ((secondary,), (primary, secondary, primary)): + with pytest.raises(ValueError, match="exactly one active occurrence"): + ArtifactSpecCollection(specs).stack_broadcast_source_indices() + + self_broadcast = replace( + primary, + relations=(InputStackBroadcastSourceRelation(source=primary.ref()),), + ) + with pytest.raises(ValueError, match="cannot broadcast from itself"): + ArtifactSpecCollection((self_broadcast,)).stack_broadcast_source_indices() + + other = ArtifactSpec.input("Other", ImageArtifactType) + multiple_owners = replace( + secondary, + relations=( + InputStackBroadcastSourceRelation(source=primary.ref()), + InputStackBroadcastSourceRelation(source=other.ref()), + ), + ) + with pytest.raises(ValueError, match="multiple stack-broadcast owners"): + ArtifactSpecCollection( + (primary, other, multiple_owners) + ).stack_broadcast_source_indices() + + def test_artifact_spec_collection_deduplicates_or_fails_loudly() -> None: image = ArtifactSpec.output("DNA", ImageArtifactType) diff --git a/tests/unit/test_automatic_image_role_materialization_435.py b/tests/unit/test_automatic_image_role_materialization_435.py new file mode 100644 index 0000000000..58f6253db9 --- /dev/null +++ b/tests/unit/test_automatic_image_role_materialization_435.py @@ -0,0 +1,378 @@ +"""Default retained image artifacts keep their declared output role identity.""" + +from dataclasses import replace +from pathlib import Path +import json + +import pytest +import numpy as np +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.memory import MemoryStorageBackend + +from test_function_outputs import context_stub, function_step_plan, record_output_path +from openhcs.constants.constants import Backend, Microscope, VariableComponents +from openhcs.core.aligned_image_payload import AlignedImageSliceContext, ImagePayloadBundleContext +from openhcs.core.artifacts import ArtifactOutputPlan, ArtifactSpec, ImageArtifactType +from openhcs.core.compiled_step_plan import RuntimeArtifactMaterializationPlan +from openhcs.core.pipeline.artifact_planning import ArtifactOutputMaterializationPlanner +from openhcs.core.runtime_artifact_values import RuntimeValue +from openhcs.core.runtime_image_values import ImageMetadataPayload, ImagePayloadMetadata, ImagePayloadMetadataCompositionMode +from openhcs.core.runtime_stores import RuntimeValueStore +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection +from openhcs.core.steps.function_output_identity import ( + FunctionOutputIdentityAuthority, FunctionOutputPathAuthority, +) +from openhcs.core.steps.function_outputs import ( + PrimaryImageMetadataTarget, RuntimeArtifactMaterializationAuthority, +) +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser + + +@pytest.mark.parametrize("plane_count", (1, 2)) +def test_automatic_two_role_images_publish_their_actual_saved_occurrences(tmp_path, plane_count): + filemanager = FileManager({ + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + }) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + context.microscope_handler.microscope_type = Microscope.OPENHCS.value + context.runtime_value_store = RuntimeValueStore() + context.metadata_cache = {} + context.tiff_config = None + plan = function_step_plan( + "Declared raw and capped roles", + variable_components=((VariableComponents.Z_INDEX,) if plane_count > 1 else ()), + ) + plan.streaming_configs = {} + plan.write_backend = Backend.DISK.value + plan.output_plate_root = str(tmp_path) + plan.output_dir = tmp_path / "images" + plan.sub_dir = "images" + plan.analysis_results_dir = str(plan.output_dir) + plan.runtime_artifact_materialization = RuntimeArtifactMaterializationPlan( + persistent_enabled=True, persistent_backend=Backend.DISK.value, + ) + context.step_plans = {plan.step_index: plan} + plans = tuple( + ArtifactOutputPlan( + name=name, path=f"/memory/{name}.pkl", artifact_type=ImageArtifactType, + producer_step_index=plan.step_index, + producer_step_scope_id=plan.step_scope_id, + producer_step_name=plan.step_name, + materialization=ArtifactOutputMaterializationPlanner.materialization_for( + ArtifactSpec(name=name, artifact_type=ImageArtifactType, plan_type=ArtifactOutputPlan), (), + ), + ) + for name in ("raw_body_processing_role", "capped_outgrowth_processing_role") + ) + plan.artifact_outputs = {output.ref(): output for output in plans} + contexts = AlignedImageSliceContext.main_flow_for_output_plans(plans) + expected = {} + for output, output_context, factor in zip(plans, contexts, (1.0, 0.5), strict=True): + payloads = tuple( + ImageMetadataPayload( + (np.arange(20, dtype=np.float32).reshape(4, 5) + index) * factor, + ImagePayloadMetadata( + source_component_metadata={ + "well": "A01", "site": "1", "channel": "2", "z_index": str(index), + "timepoint": "1", "extension": ".tif", + }, + source_image_names=(output.name,), + source_voxel_spacing=SourceVoxelSpacing((1.3556, 1.3556)), + ), + ) + for index in range(1, plane_count + 1) + ) + value = ( + payloads[0] if plane_count == 1 else ImagePayloadBundleContext.from_payloads( + payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + ).compose() + ) + context.runtime_value_store.record( + RuntimeValue.normalize(output, value, axis_id="A01"), + path=output.path, backend=Backend.MEMORY.value, + ) + expected[output.name] = [] + for payload in payloads: + identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata( + context.microscope_handler.parser, payload.metadata, + ).with_filename_qualifier(output_context.output_key) + destination = plan.output_dir / FunctionOutputPathAuthority.filename_for_identity( + context.microscope_handler.parser, identity, + ) + filemanager.ensure_directory(plan.output_dir, Backend.MEMORY.value) + filemanager.ensure_directory(plan.output_dir, Backend.DISK.value) + filemanager.save(payload, str(destination), Backend.MEMORY.value) + filemanager.save(payload.data, str(destination), Backend.DISK.value) + record_output_path( + context, plan, str(destination), output_context=output_context, + image_metadata=payload.metadata, identity=identity, + ) + expected[output.name].append((destination, payload.data)) + saved = RuntimeArtifactMaterializationAuthority.materialize(context, plan) + target = replace(PrimaryImageMetadataTarget.from_plan(plan), artifact_materializations=saved) + # The original implementation raises its real duplicate projection guard + # here: both automatic materializations used the same unqualified path. + structured = target.produced_projection_metadata(context, plan) + assert {item["source_alias"] for item in structured["source_projection"]} == set(expected) + assert len(structured["source_projection"]) == 2 * plane_count + for materialized in saved: + outputs = materialized.outputs_for_backend(Backend.DISK.value) + assert len(outputs) == plane_count + for actual, (path, pixels) in zip(outputs, expected[materialized.materialization.output_plan.name], strict=True): + assert actual.path == str(path) + np.testing.assert_array_equal(tifffile.imread(path), pixels) + target.write(context, produced_plan=plan) + document = json.loads((tmp_path / "openhcs_metadata.json").read_text()) + reopened = VirtualWorkspaceSourceProjection.from_openhcs_metadata(tmp_path, document) + for name, expected_planes in expected.items(): + for path, pixels in expected_planes: + projection = reopened.source_projections_by_virtual_path[str(path.relative_to(tmp_path))] + assert projection.source_alias == name + assert projection.address.as_component_metadata()["channel"] == "2" + assert projection.image_metadata.source_voxel_spacing == SourceVoxelSpacing((1.3556, 1.3556)) + np.testing.assert_array_equal(tifffile.imread(tmp_path / projection.ref.backend_address), pixels) + + +@pytest.mark.parametrize("purpose", ( + "explicit_source", "explicit_template", "terminal_template", + "explicit_artifact", "terminal_artifact", "unnamed_terminal", "unnamed_explicit", + "explicit_source_declared_terminal", "terminal_source_declared_explicit", +)) +def test_retained_role_policy_preserves_authored_and_unnamed_image_paths(tmp_path, purpose): + from openhcs.processing.materialization import ( + ImageFileOptions, MaterializationSpec, MaterializedFilenameIdentity, + TerminalMaterializationSpec, materialization_outputs, + ) + + options = ImageFileOptions( + filename_suffix=".tif", + relative_path_template=("authored/plane_{index}.tif" if purpose.endswith("template") else None), + filename_identity=( + MaterializedFilenameIdentity.ARTIFACT_NAME if purpose.endswith("artifact") + else MaterializedFilenameIdentity.SOURCE_IDENTITY + ), + ) + spec = ( + TerminalMaterializationSpec(options) if purpose.startswith("terminal") or purpose == "unnamed_terminal" + else MaterializationSpec(options) + ) + declared_spec = ( + TerminalMaterializationSpec(options) if purpose == "explicit_source_declared_terminal" + else MaterializationSpec(options) if purpose == "terminal_source_declared_explicit" + else spec + ) + output_plan = None if purpose in ("unnamed_terminal", "unnamed_explicit") else ArtifactOutputPlan( + name="DeclaredRole", path="/memory/DeclaredRole.pkl", + artifact_type=ImageArtifactType, materialization=declared_spec, + ) + payloads = tuple( + ImageMetadataPayload(np.ones((4, 5), dtype=np.float32) * index, ImagePayloadMetadata( + source_component_metadata={ + "well": "A01", "site": "1", "channel": "2", "z_index": str(index), + "timepoint": "1", "extension": ".tif", + }, + )) + for index in (1, 2) + ) + value = ImagePayloadBundleContext.from_payloads( + payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + ).compose() + filemanager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + outputs = materialization_outputs( + spec, value, str(tmp_path / "AuthoredBase"), filemanager, + context=context, output_plan=output_plan, + variable_components=(VariableComponents.Z_INDEX,), + ) + expected_paths = ( + ("authored/plane_1.tif", "authored/plane_2.tif") if purpose.endswith("template") + else ("AuthoredBase.tif",) if purpose.endswith("artifact") + else ("A01_s001_w2_z001_t001_DeclaredRole.tif", "A01_s001_w2_z002_t001_DeclaredRole.tif") if purpose == "terminal_source_declared_explicit" + else ("A01_s001_w2_z001_t001.tif", "A01_s001_w2_z002_t001.tif") + ) + assert tuple(Path(output.path).relative_to(tmp_path).as_posix() for output in outputs) == expected_paths + if len(outputs) == 2: + for actual, expected in zip(outputs, payloads, strict=True): + np.testing.assert_array_equal(actual.content, expected.data) + else: + np.testing.assert_array_equal(outputs[0].content, value.data) + + +def test_authored_duplicate_template_still_fails_before_any_save(tmp_path): + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec, materialization_outputs + + spec = TerminalMaterializationSpec(ImageFileOptions( + filename_suffix=".tif", relative_path_template="same.tif", + )) + output_plan = ArtifactOutputPlan(name="NamedRole", path="/memory/role.pkl", artifact_type=ImageArtifactType, materialization=spec) + payloads = tuple( + ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": str(index), "timepoint": "1", "extension": ".tif"}, + )) for index in (1, 2) + ) + value = ImagePayloadBundleContext.from_payloads(payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK).compose() + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + with pytest.raises(ValueError, match="produced duplicate paths"): + materialization_outputs(spec, value, str(tmp_path / "role"), manager, + context=context_stub(manager, parser=SourceSchemaFilenameParser()), + output_plan=output_plan) + assert not (tmp_path / "same.tif").exists() + + +@pytest.mark.parametrize("suffix", (".tif", ".png", ".ome.tif", ".labels.tif")) +@pytest.mark.parametrize("plane_count", (1, 2)) +def test_retained_image_qualifier_respects_complete_writer_suffix(tmp_path, suffix, plane_count): + from openhcs.core.steps.function_artifact_materialization import RuntimeArtifactMaterialization + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + context = context_stub(manager, parser=SourceSchemaFilenameParser()) + plan = function_step_plan("Retained format role", variable_components=(VariableComponents.Z_INDEX,) if plane_count > 1 else ()) + plan.output_dir = tmp_path / "images" + plan.analysis_results_dir = str(plan.output_dir) + output_plan = ArtifactOutputPlan( + name="Role Name", path="/memory/role.pkl", artifact_type=ImageArtifactType, + materialization=TerminalMaterializationSpec(ImageFileOptions(filename_suffix=suffix)), + ) + payloads = tuple( + ImageMetadataPayload(np.ones((4, 5), dtype=np.uint8) * index, ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": str(index), "timepoint": "1", "extension": ".tif"}, + )) for index in range(1, plane_count + 1) + ) + value = payloads[0] if plane_count == 1 else ImagePayloadBundleContext.from_payloads( + payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + ).compose() + store = RuntimeValueStore() + record = store.record(RuntimeValue.normalize(output_plan, value, axis_id="A01"), path=output_plan.path, backend=Backend.MEMORY.value) + outputs = RuntimeArtifactMaterialization.from_record( + output_plan=output_plan, record=record, plan=plan, context=context, + ).outputs(plan, context) + assert len(outputs) == plane_count + for index, (output, payload) in enumerate(zip(outputs, payloads, strict=True), 1): + identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata( + context.microscope_handler.parser, payload.metadata, + ).with_filename_qualifier(output_plan.name) + expected = FunctionOutputPathAuthority.filename_for_identity( + context.microscope_handler.parser, replace(identity, extension=suffix), + ) + assert Path(output.path).name == expected + assert Path(output.path).name.endswith("_Role_Name" + suffix) + assert output.metadata.source_component_metadata["channel"] == "2" + assert output.metadata.source_component_metadata["z_index"] == str(index) + np.testing.assert_array_equal(output.content, payload.data) + + +def test_distinct_aliases_that_normalize_to_one_filename_still_conflict(tmp_path): + from openhcs.core.steps.function_artifact_materialization import RuntimeArtifactMaterialization + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + context = context_stub(manager, parser=SourceSchemaFilenameParser()) + plan = function_step_plan("Ambiguous role spelling") + plan.output_dir = tmp_path / "images" + plan.analysis_results_dir = str(plan.output_dir) + value = ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": "1", "timepoint": "1", "extension": ".tif"}, + )) + paths = [] + store = RuntimeValueStore() + for name in ("a/b", "a b"): + output_plan = ArtifactOutputPlan(name=name, path=f"/memory/{name}.pkl", artifact_type=ImageArtifactType, + materialization=TerminalMaterializationSpec(ImageFileOptions(filename_suffix=".tif"))) + record = store.record(RuntimeValue.normalize(output_plan, value, axis_id="A01"), path=output_plan.path, backend=Backend.MEMORY.value) + outputs = RuntimeArtifactMaterialization.from_record(output_plan=output_plan, record=record, plan=plan, context=context).outputs(plan, context) + paths.append(outputs[0].path) + assert paths[0] == paths[1] + # Existing main-flow destination authority rejects this declared naming + # collision instead of choosing a new suffix or inventing a channel. + from openhcs.core.steps.function_runtime import OutputPathBatchUniqueness + with pytest.raises(ValueError, match="duplicate"): + OutputPathBatchUniqueness(output_paths=paths, input_paths=[], step_name=plan.step_name, pattern_repr="declared roles").validate() + + +def test_batch_binds_actual_purpose_once_without_rewriting_plan_or_source(monkeypatch, tmp_path): + from openhcs.processing.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec + from openhcs.processing.materialization.constants import MaterializationFormat + from openhcs.processing.materialization.core import MaterializationBatch, MaterializationContext, Output, WriterSpec, _WRITERS_BY_OPTIONS + + options = ImageFileOptions(filename_suffix=".tif") + declared = TerminalMaterializationSpec(options) + explicit = MaterializationSpec(options) + plan = ArtifactOutputPlan(name="Role", path="/memory/role", artifact_type=ImageArtifactType, materialization=declared) + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + source = {"physical_source": "original"} + context = MaterializationContext( + base_path=str(tmp_path / "role"), backends=[], backend_kwargs={}, + filemanager=manager, extra_inputs=source, output_plan=plan, + materialization_spec=declared, + ) + seen = [] + def custom_writer(data, actual_options, actual_context): + assert actual_options is options + assert actual_context.output_plan is plan + assert actual_context.extra_inputs is source + seen.append(actual_context) + return [Output(path=str(tmp_path / "custom.tif"), content=data)] + monkeypatch.setitem(_WRITERS_BY_OPTIONS, ImageFileOptions, WriterSpec( + format=MaterializationFormat.IMAGE_FILE, options_type=ImageFileOptions, + write=custom_writer, primary_path=lambda outputs: outputs[0].path, + candidate_paths=lambda _options, base: (base,), + )) + pixels = np.ones((4, 5), dtype=np.float32) + first = MaterializationBatch.render(explicit, pixels, context) + second = MaterializationBatch.render(declared, pixels, first.context) + assert seen == [first.context, second.context] + assert seen[0] is first.context and seen[1] is second.context + assert first.context.materialization_spec is explicit + assert second.context.materialization_spec is declared + assert context.materialization_spec is declared + assert plan.materialization is declared + assert source == {"physical_source": "original"} + + +def test_direct_image_writer_without_purpose_binding_retains_legacy_source_path(tmp_path): + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + from openhcs.processing.materialization.core import MaterializationContext, write_image_file + + options = ImageFileOptions(filename_suffix=".tif") + plan = ArtifactOutputPlan(name="Role", path="/memory/role", artifact_type=ImageArtifactType, + materialization=TerminalMaterializationSpec(options)) + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + value = ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": "1", "timepoint": "1", "extension": ".tif"}, + )) + context = MaterializationContext(base_path=str(tmp_path / "role"), backends=[], backend_kwargs={}, + filemanager=manager, extra_inputs={}, output_plan=plan, + context=context_stub(manager, parser=SourceSchemaFilenameParser())) + outputs = write_image_file(value, options, context) + assert Path(outputs[0].path).name == "A01_s001_w2_z001_t001.tif" + assert context.materialization_spec is None + assert context.output_plan is plan + np.testing.assert_array_equal(outputs[0].content, value.data) + + +@pytest.mark.parametrize("purpose", ("named_terminal", "explicit", "unnamed_terminal")) +def test_parser_free_render_requires_parser_only_for_automatic_named_role(tmp_path, purpose): + from openhcs.processing.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec, materialization_outputs + + options = ImageFileOptions(filename_suffix=".tif") + spec = MaterializationSpec(options) if purpose == "explicit" else TerminalMaterializationSpec(options) + plan = None if purpose == "unnamed_terminal" else ArtifactOutputPlan( + name="Role", path="/memory/role", artifact_type=ImageArtifactType, materialization=spec, + ) + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + value = ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_path="/physical/OriginalSource.tif", + )) + if purpose == "named_terminal": + with pytest.raises(ValueError, match="Parser-backed source-stem authority requires a parser"): + materialization_outputs(spec, value, str(tmp_path / "Role"), manager, output_plan=plan) + else: + outputs = materialization_outputs(spec, value, str(tmp_path / "Role"), manager, output_plan=plan) + assert Path(outputs[0].path).name == "OriginalSource.tif" + np.testing.assert_array_equal(outputs[0].content, value.data) + assert value.metadata.source_path == "/physical/OriginalSource.tif" diff --git a/tests/unit/test_callable_abi_preparation.py b/tests/unit/test_callable_abi_preparation.py new file mode 100644 index 0000000000..dd3f1aa8e9 --- /dev/null +++ b/tests/unit/test_callable_abi_preparation.py @@ -0,0 +1,401 @@ +"""Prepared canonical signatures survive compilation and worker transport.""" + +import inspect +import pickle +from types import SimpleNamespace +from dataclasses import replace +from functools import wraps +from unittest.mock import Mock +from typing import Any + +import cloudpickle +import numpy as np +import pytest +from arraybridge import ArrayPayload + +import openhcs.core.callable_contract as contract_module +from openhcs.core.callable_contract import ( + CallableContract, CallableImportIdentity, CallableMetadata, CallableProjection, + attach_callable_contract_metadata, attach_processing_prepare, +) +from openhcs.core.function_contract_metadata import FunctionContractAttribute +from openhcs.core.function_patterns import NormalizeFunctionGroupAuthority +from openhcs.core.function_reference import ImportableFunctionReference +from openhcs.core.processing_preparation import CallablePreparation, PreparationCacheBatch +from openhcs.core.runtime_image_values import ImagePayloadMetadata, image_payload_data +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.pipeline.function_contracts import composed_image_payload +from openhcs.processing.backends.lib_registry.registry_service import RegistryService +from openhcs.core.runtime_batch_contracts import ( + Pure2DSliceBatchExecutor, RuntimeBatchExecutionDomain, + SerialPure2DSliceBatchExecutor, runtime_callable_defaults, +) +from openhcs.core.pipeline.function_contracts import resolved_callable_parameter +from openhcs.processing.backends.lib_registry.unified_registry import ( + RuntimeCallablePolicy, RuntimeInvocationKwargPolicy, +) + + +def raw_numeric(image: np.ndarray, scale: float = 2.0) -> np.ndarray: + return image * scale + + +@composed_image_payload +def unannotated_composed_echo(image): + return image + + +@pytest.mark.parametrize( + ("composed", "annotation", "retains_payload"), + ((True, inspect.Parameter.empty, True), + (False, inspect.Parameter.empty, False), + (True, np.ndarray, False), + (True, Any, False), + (True, ArrayPayload, True)), +) +def test_composition_default_respects_explicit_raw_carrier_boundary( + composed, annotation, retains_payload, fake_preparation, +): + received = [] + def raw(image): + received.append(image) + return image + if annotation is not inspect.Parameter.empty: + raw.__annotations__["image"] = annotation + if composed: + composed_image_payload(raw) + contract = CallableContract.from_prepared_callable(raw) + pixels = np.arange(20, dtype=np.float32).reshape(1, 4, 5) + mask = np.ones((1, 4, 5), dtype=bool) + mask[0, 0, 0] = False + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_image_names=("FITC",), + ).payload_with(pixels, mask) + expected = source if retains_payload else pixels + + result = RuntimeCallablePolicy().contract_invocation(contract, raw, source, {}).call() + + assert result is expected + assert received[0] is expected + assert source.metadata.plane_axis is RuntimePlaneAxis.SOURCE_BINDING + np.testing.assert_array_equal(source.mask, mask) + + +@pytest.mark.parametrize("serializer", (pickle, cloudpickle)) +def test_prepared_composition_default_survives_transport_without_introspection( + serializer, fake_preparation, monkeypatch, +): + contract = CallableContract.from_prepared_callable(unannotated_composed_echo) + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(unannotated_composed_echo), + composite_key="test:unannotated_composed_echo", metadata=contract.metadata, + ) + restored = CallableContract.from_callable(serializer.loads(serializer.dumps(reference))) + def forbidden(*args, **kwargs): + pytest.fail("Prepared composed ABI must not inspect live callable declarations") + monkeypatch.setattr(contract_module, "get_type_hints", forbidden) + monkeypatch.setattr(inspect, "signature", forbidden) + payload = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_image_names=("FITC",), + ).payload_with(np.ones((1, 4, 5)), np.ones((1, 4, 5), dtype=bool)) + for _ in range(40): + assert restored.raw_main_flow_call_argument(payload) is payload + + +@pytest.fixture +def fake_preparation(monkeypatch): + prepare = Mock() + monkeypatch.setattr(CallablePreparation, "prepare", prepare) + monkeypatch.setattr(PreparationCacheBatch, "populate_child_caches", lambda *a, **k: None) + return prepare + + +def test_registry_resolves_signatures_after_all_hooks_and_before_ready(monkeypatch): + events = [] + + def first(image: np.ndarray): + return image + + def last(image: np.ndarray): + return image + + first.__dict__[FunctionContractAttribute.raw_processing_function] = last + metadata = {"first": SimpleNamespace(func=first), "alias": SimpleNamespace(func=first)} + monkeypatch.setattr(RegistryService, "_metadata_cache", metadata) + monkeypatch.setattr(PreparationCacheBatch, "populate_child_caches", lambda *a, **k: None) + + def prepare(owner): + events.append(("prepare", owner.projection.func)) + if owner.projection.func is last: + last.__annotations__["image"] = ArrayPayload + + original_hints = contract_module.get_type_hints + + def hints(func, **kwargs): + events.append(("signature", func)) + return original_hints(func, **kwargs) + + monkeypatch.setattr(CallablePreparation, "prepare", prepare) + monkeypatch.setattr(contract_module, "get_type_hints", hints) + assert RegistryService.prepare_in_current_process(status_callback=events.append) is metadata + preparations = [i for i, item in enumerate(events) if isinstance(item, tuple) and item[0] == "prepare"] + signatures = [i for i, item in enumerate(events) if isinstance(item, tuple) and item[0] == "signature"] + assert len(preparations) == 2 + assert max(preparations) < min(signatures) + assert max(signatures) < next(i for i, item in enumerate(events) if isinstance(item, str) and "kernels ready" in item) + assert CallableContract.from_callable(first).canonical_parameter_annotations["image"] is ArrayPayload + assert CallableContract.from_callable(last).canonical_parameter_annotations["image"] is ArrayPayload + + +def test_explicit_library_rewarm_refreshes_new_contract_not_existing_snapshot(monkeypatch, fake_preparation): + def raw(image: np.ndarray): + return image + + metadata = {"raw": SimpleNamespace(func=raw)} + monkeypatch.setattr(RegistryService, "_metadata_cache", metadata) + RegistryService.prepare_in_current_process() + first = CallableContract.from_callable(raw) + raw.__annotations__["image"] = ArrayPayload + assert CallableContract.from_callable(raw).canonical_parameter_annotations["image"] is np.ndarray + RegistryService.prepare_in_current_process() + second = CallableContract.from_callable(raw) + assert first.canonical_parameter_annotations["image"] is np.ndarray + assert second.canonical_parameter_annotations["image"] is ArrayPayload + + +def test_authored_compilation_prepares_before_signature_and_refreshes_each_compilation(monkeypatch): + def raw(image: np.ndarray, scale=2): + return image + + desired_annotations = [ArrayPayload, np.ndarray] + + def prepare(owner): + raw.__annotations__["image"] = desired_annotations.pop(0) + + monkeypatch.setattr(CallablePreparation, "prepare", prepare) + first = NormalizeFunctionGroupAuthority().normalize("group", raw).items[0].contract + second = NormalizeFunctionGroupAuthority().normalize("group", raw).items[0].contract + assert first.canonical_parameter_annotations["image"] is ArrayPayload + assert second.canonical_parameter_annotations["image"] is np.ndarray + assert FunctionContractAttribute.canonical_signature not in vars(raw) + + +def test_authored_preparation_precedes_metadata_validation(monkeypatch): + def raw(image: np.ndarray): + return image + events = [] + original = CallableMetadata.from_projection.__func__ + + def validate(cls, projection): + assert events == ["prepared"] + return original(cls, projection) + + monkeypatch.setattr(CallablePreparation, "prepare", lambda self: events.append("prepared")) + monkeypatch.setattr(CallableMetadata, "from_projection", classmethod(validate)) + contract = CallableContract.from_prepared_callable(raw) + assert contract.canonical_parameter_annotations["image"] is np.ndarray + + +def test_unprepared_function_reference_compilation_reads_prepared_live_declaration(monkeypatch, fake_preparation): + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(raw_numeric), + composite_key="test:raw_numeric", + ) + contract = NormalizeFunctionGroupAuthority().normalize("group", reference).items[0].contract + assert contract.func is reference + assert contract.canonical_parameter_annotations["image"] is np.ndarray + assert fake_preparation.call_count == 1 + + +@pytest.mark.parametrize("serializer", [pickle, cloudpickle]) +def test_prepared_signature_metadata_namespace_and_reference_transport(serializer, fake_preparation): + contract = CallableContract.from_prepared_callable(raw_numeric) + metadata = contract.metadata + + def proxy(image): + return image + + proxy.__dict__.update(metadata.as_namespace()) + assert CallableMetadata.from_callable(proxy).canonical_signature == metadata.canonical_signature + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(raw_numeric), + composite_key="test:raw_numeric", metadata=metadata, + ) + restored = serializer.loads(serializer.dumps(reference)) + assert restored.metadata.canonical_signature == metadata.canonical_signature + restored_contract = CallableContract.from_callable(restored) + assert restored_contract.canonical_parameter_annotations["image"] is np.ndarray + assert restored_contract.canonical_signature.parameters["scale"].default == 2.0 + assert restored_contract.canonical_signature.return_annotation is np.ndarray + + +def test_compiled_raw_projection_never_queries_signature_or_annotations(monkeypatch, fake_preparation): + contract = CallableContract.from_prepared_callable(raw_numeric) + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(raw_numeric), + composite_key="test:raw_numeric", metadata=contract.metadata, + ) + restored = CallableContract.from_callable(pickle.loads(pickle.dumps(reference))) + def forbidden(*args, **kwargs): + raise AssertionError("Compiled runtime queried a live signature") + monkeypatch.setattr(contract_module, "get_type_hints", forbidden) + monkeypatch.setattr(inspect, "signature", forbidden) + payload = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with(np.ones((2, 3)), None) + for _ in range(40): + assert restored.raw_main_flow_call_argument(payload) is image_payload_data(payload) + assert restored.primary_input_parameter_name == "image" + assert restored.canonical_parameter_annotations["image"] is np.ndarray + + +def test_custom_signature_and_mutable_default_are_retained_until_recompilation(monkeypatch, fake_preparation): + default = [] + def raw(image, values=None): + return image + raw.__signature__ = inspect.Signature([ + inspect.Parameter("image", inspect.Parameter.POSITIONAL_ONLY, annotation=ArrayPayload), + inspect.Parameter("values", inspect.Parameter.KEYWORD_ONLY, default=default), + ]) + first = CallableContract.from_prepared_callable(raw) + assert first.canonical_signature.parameters["values"].default is default + raw.__signature__ = raw.__signature__.replace(parameters=[ + inspect.Parameter("image", inspect.Parameter.POSITIONAL_ONLY, annotation=np.ndarray), + ]) + second = CallableContract.from_prepared_callable(raw) + assert first.canonical_parameter_annotations["image"] is ArrayPayload + assert second.canonical_parameter_annotations["image"] is np.ndarray + + +@pytest.mark.parametrize("mutation", [ + lambda func: attach_processing_prepare(func, lambda: None), + lambda func: attach_callable_contract_metadata(func, raw_processing_function=raw_numeric), +]) +def test_supported_declaration_changes_invalidate_published_signature(mutation): + def raw(image: np.ndarray): + return image + CallableProjection.from_callable(raw).warm_canonical_signature() + assert FunctionContractAttribute.canonical_signature in vars(raw) + mutation(raw) + assert FunctionContractAttribute.canonical_signature not in vars(raw) + + +def test_malformed_signature_declaration_is_rejected(): + def raw(image): + return image + raw.__dict__[FunctionContractAttribute.canonical_signature] = "not a signature" + with pytest.raises(TypeError, match="inspect.Signature"): + CallableMetadata.from_callable(raw) + + +def test_exact_raw_default_filter_and_type_views_refresh_with_library_snapshot(): + def raw(image: np.ndarray, scale: float = 2): + return image * scale + CallableProjection.from_callable(raw).warm_canonical_signature() + policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) + assert runtime_callable_defaults(raw)["scale"] == 2 + assert resolved_callable_parameter(raw, "scale").annotation is float + assert policy.invocation(raw, (np.ones((2, 3)),), {"scale":3, "unknown":9}).call()[0,0] == 3 + raw.__defaults__ = (4,) + raw.__annotations__["scale"] = int + assert runtime_callable_defaults(raw)["scale"] == 2 + assert resolved_callable_parameter(raw, "scale").annotation is float + CallableProjection.from_callable(raw).warm_canonical_signature() + assert runtime_callable_defaults(raw)["scale"] == 4 + assert resolved_callable_parameter(raw, "scale").annotation is int + + +def test_different_wrapper_signature_remains_live_for_default_filter_and_parameter_queries(): + def raw(image: np.ndarray, private: int = 8): + return image + def wrapper(image: np.ndarray, public: int = 2): + return image * public + wrapper.__dict__[FunctionContractAttribute.raw_processing_function] = raw + CallableProjection.from_callable(wrapper).warm_canonical_signature() + assert runtime_callable_defaults(wrapper) == {"public":2} + assert resolved_callable_parameter(wrapper, "public").annotation is int + policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) + assert policy.invocation(wrapper, (np.ones((2, 3)),), {"public":3, "private":99}).call()[0,0] == 3 + wrapper.__defaults__ = (5,) + assert runtime_callable_defaults(wrapper) == {"public":5} + assert policy.invocation(wrapper, (np.ones((2, 3)),), {"private":99}).call()[0,0] == 5 + + +def test_prepared_type_query_preserves_legacy_nested_annotation_stripping(): + from typing import Annotated, get_type_hints + def raw(image, labels): + return image + raw.__annotations__ = {"image":Annotated[np.ndarray,"image"],"labels":tuple[Annotated[int,"id"],...],"return":Annotated[np.ndarray,"result"]} + expected = get_type_hints(raw) + CallableProjection.from_callable(raw).warm_canonical_signature() + assert resolved_callable_parameter(raw,"image").annotation == expected["image"] + assert resolved_callable_parameter(raw,"labels").annotation == expected["labels"] + assert CallableContract.from_callable(raw).canonical_parameter_annotations["labels"] == raw.__annotations__["labels"] + + +def test_distinct_authored_runtime_signature_transport_and_new_compilation_refresh(fake_preparation): + def raw(image: np.ndarray, scale: int = 2) -> np.ndarray: + return image * scale + @wraps(raw) + def wrapper(*args, **kwargs): + return raw(*args, **kwargs) + wrapper.__signature__ = inspect.signature(raw).replace(parameters=[ + *inspect.signature(raw).parameters.values(), + inspect.Parameter("injected",inspect.Parameter.KEYWORD_ONLY,default=False), + ]) + first = CallableContract.from_prepared_callable(wrapper) + assert "injected" in first.canonical_signature.parameters + assert "injected" not in first.raw_runtime_signature.parameters + assert first.metadata.raw_runtime_signature is not None + restored = pickle.loads(pickle.dumps(first.metadata)) + assert restored.raw_runtime_signature == first.raw_runtime_signature + assert restored.as_namespace()[FunctionContractAttribute.raw_runtime_signature] == first.raw_runtime_signature + assert FunctionContractAttribute.canonical_signature not in vars(raw) + assert FunctionContractAttribute.canonical_signature not in vars(wrapper) + raw.__defaults__ = (4,) + raw.__annotations__["scale"] = float + second = CallableContract.from_prepared_callable(wrapper) + assert first.raw_runtime_signature.parameters["scale"].default == 2 + assert second.raw_runtime_signature.parameters["scale"].default == 4 + assert second.raw_runtime_signature.parameters["scale"].annotation is float + + +def test_distinct_targets_own_signatures_even_when_layouts_match_without_comparing_defaults(fake_preparation): + class RejectEquality: + def __eq__(self, other): + raise AssertionError("Signature reuse compared a mutable default") + default = RejectEquality() + def raw(image: np.ndarray, values=default) -> np.ndarray: + return image + @wraps(raw) + def wrapper(*args, **kwargs): + return raw(*args, **kwargs) + contract = CallableContract.from_prepared_callable(wrapper) + assert contract.metadata.raw_runtime_signature is not None + assert contract.raw_runtime_signature is not contract.canonical_signature + assert contract.raw_runtime_signature.parameters["values"].default is default + raw.__defaults__ = ("new raw default",) + second = CallableContract.from_prepared_callable(wrapper) + assert second.raw_runtime_signature.parameters["values"].default == "new raw default" + assert contract.raw_runtime_signature.parameters["values"].default is default + + +def test_same_actual_target_shares_canonical_signature(fake_preparation): + contract = CallableContract.from_prepared_callable(raw_numeric) + assert contract.metadata.raw_runtime_signature is None + assert contract.raw_runtime_signature is contract.canonical_signature + + +@pytest.mark.parametrize("executors", (None, {}, {RuntimeBatchExecutionDomain.PURE_2D_SLICES: None})) +def test_pure_2d_executor_selection_retains_serial_fallback(executors): + assert isinstance(Pure2DSliceBatchExecutor.from_executors(executors), SerialPure2DSliceBatchExecutor) + + +def test_pure_2d_executor_selection_retains_exact_declared_executor(): + def declared(*args, **kwargs): + raise AssertionError("Selecting an executor must not execute it") + + selected = Pure2DSliceBatchExecutor.from_executors({ + RuntimeBatchExecutionDomain.PURE_2D_SLICES: declared, + }) + assert selected is declared diff --git a/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py b/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py new file mode 100644 index 0000000000..1ceb91da58 --- /dev/null +++ b/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py @@ -0,0 +1,234 @@ +"""Prepared raw arguments retain context consumed by nominal CP request owners.""" +from dataclasses import replace + +import numpy as np +import pytest + +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.callable_contract import CallableContract +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, + normalize_image_payload_intensity, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter +from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( + CellProfilerFunctionContractExecutor, +) +from openhcs.processing.backends.cellprofiler.alignment import AlignModule, align +from openhcs.processing.backends.cellprofiler.area_occupied import measure_image_area_occupied +from openhcs.processing.backends.cellprofiler.crop import CropModule, crop +from openhcs.processing.backends.cellprofiler.grid import ShapeChoice, identify_objects_in_grid +from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest +from openhcs.core.runtime_spatial_grid import SpatialGrid +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.processing.backends.cellprofiler.image_quality import measure_image_quality +from openhcs.processing.backends.cellprofiler.intensity import measure_object_intensity +from openhcs.processing.backends.cellprofiler.manual_objects import identify_objects_manually +from openhcs.processing.backends.cellprofiler.object_images import convert_image_to_objects +from openhcs.processing.backends.cellprofiler.thresholding import threshold +from openhcs.processing.backends.cellprofiler.worms import ( + identify_dead_worms, + untangle_worms, + untangle_worms_both, + untangle_worms_with_overlap, +) + + +def _prepared_contract(func): + contract = CallableContract.from_callable(func).with_prepared_signature() + return replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + + +@pytest.mark.parametrize( + "func", + ( + align, + crop, + measure_image_area_occupied, + measure_object_intensity, + measure_image_quality, + identify_objects_manually, + identify_objects_in_grid, + convert_image_to_objects, + identify_dead_worms, + threshold, + untangle_worms, + untangle_worms_with_overlap, + untangle_worms_both, + ), +) +def test_prepared_primary_context_consumers_keep_nominal_payload_and_bare_pixels(func): + pixels = np.arange(60, dtype=np.float32).reshape(4, 5, 3) + mask = np.ones((4, 5), dtype=bool) + mask[0] = False + source = ImagePayloadMetadata( + source_path="/input/rgb.tif", + source_image_names=("RGB",), + source_channel_axis=-1, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(8, 10) + ), + ).payload_with(pixels, mask) + contract = _prepared_contract(func) + + projected = contract.raw_main_flow_call_argument(source) + + assert projected is source + assert image_payload_mask(projected) is mask + assert image_payload_metadata(projected).source_spatial_domain.origin_yx == (2, 3) + assert image_payload_metadata(projected).source_image_paths == ("/input/rgb.tif",) + assert contract.raw_main_flow_call_argument(pixels) is pixels + + +@pytest.mark.parametrize("shape", ((5, 5), (4, 7))) +@pytest.mark.parametrize("shape_choice", (ShapeChoice.RECTANGLE, ShapeChoice.NATURAL, ShapeChoice.CIRCLE_NATURAL)) +def test_prepared_grid_objects_preserve_rgb_mask_calibration_and_crop_domain(shape, shape_choice): + pixels = np.arange(np.prod(shape) * 3, dtype=np.float32).reshape(*shape, 3) + mask = np.ones(shape, dtype=bool) + mask[0] = False + metadata = ImagePayloadMetadata( + source_path="/input/grid_rgb.tif", + source_channel_axis=-1, + source_voxel_spacing=SourceVoxelSpacing((2.0, 0.7, 0.9)), + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(shape[0] + 4, shape[1] + 6), + ), + ) + source = metadata.payload_with(pixels, mask) + grid = SpatialGrid( + name="Grid", rows=1, columns=1, x_spacing=4.0, y_spacing=4.0, + x_origin=2.0, y_origin=2.0, source_spatial_shape_yx=shape, + ) + guide_pixels = np.zeros(shape, dtype=np.int32) + guide_pixels[1:3, 1:4] = 1 + guides = SourceImageObjectLabelBuildRequest(image=source, labels=guide_pixels).payload() + topology_inputs = (grid,) if shape_choice is ShapeChoice.RECTANGLE else (grid, guides) + contract = _prepared_contract(identify_objects_in_grid) + + returned_image, _rows, objects = CellProfilerFunctionContractExecutor().execute( + contract, contract.resolve_canonical_raw_callable(), source, + {"topology_inputs": topology_inputs, "shape_choice": shape_choice}, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + assert returned_image is source + assert image_payload_mask(returned_image) is mask + assert objects.source_provenance == metadata.source_provenance + assert objects.source_spatial_domain.origin_yx == (2, 3) + assert objects.source_spatial_domain.source_shape_yx == metadata.source_spatial_domain.source_shape_yx + assert objects.parent_image_source_voxel_spacing == metadata.source_voxel_spacing + assert objects.labels.shape == shape + assert objects.domain.declared_object_count == 1 + assert objects.domain.declared_object_ids == (1,) + expanded = objects.source_spatial_domain.materialize(objects.labels, spatial_axes_yx=(0, 1)) + assert expanded.shape == metadata.source_spatial_domain.source_shape_yx + np.testing.assert_array_equal(expanded[2:2 + shape[0], 3:3 + shape[1]], objects.labels) + + +@pytest.mark.parametrize("color", (False, True)) +def test_prepared_align_preserves_named_plane_axis_masks_and_source_identity(color): + shape = (8, 9, 3) if color else (8, 9) + image = np.random.default_rng(1).random(shape, dtype=np.float32) + pixels = np.stack((image, image)) + masks = np.ones((2, 8, 9), dtype=bool) + masks[:, 0] = False + metadata = ImagePayloadMetadata( + source_image_names=("First", "Second"), + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/input/first.tif", "/input/second.tif"), + component_metadata=({"well": "A01", "channel": "1"}, {"well": "A01", "channel": "2"}), + ), + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_channel_axis=3 if color else None, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(14, 17) + ), + ) + source = metadata.payload_with(pixels, masks) + contract = _prepared_contract(align) + + outputs, measurements = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + {"method": AlignModule.Method.NORMALIZED_CROSS_CORRELATION, + "crop_mode": AlignModule.CropMode.KEEP_SIZE}, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + assert len(outputs.slices) == 2 + assert measurements.row_count() == 2 + for index, output in enumerate(outputs.slices): + np.testing.assert_array_equal(image_payload_data(output), pixels[index]) + np.testing.assert_array_equal(image_payload_mask(output), masks[index]) + output_metadata = image_payload_metadata(output) + assert output_metadata.source_image_paths == (("/input/first.tif", "/input/second.tif")[index],) + assert output_metadata.source_spatial_domain == metadata.source_spatial_domain + assert output_metadata.source_channel_axis == (2 if color else None) + + +def test_prepared_crop_preserves_rgb_spatial_mask_and_calibrated_source_domain(): + pixels = np.arange(60, dtype=np.float32).reshape(4, 5, 3) + parent_mask = np.ones((4, 5), dtype=bool) + parent_mask[0] = False + domain = SourceSpatialDomain(origin_yx=(2, 3), source_shape_yx=(8, 10)) + source = ImagePayloadMetadata( + source_channel_axis=-1, + source_image_names=("RGB",), + source_path="/input/rgb.tif", + source_spatial_domain=domain, + ).payload_with(pixels, parent_mask) + contract = _prepared_contract(crop) + + output, cropping, measurements = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + {"left_right_rectangle_positions": (1, 4), + "top_bottom_rectangle_positions": (0, 3), + "removal_method": CropModule.RemovalMethod.NO}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + ) + + expected_crop = np.zeros((4, 5), dtype=bool) + expected_crop[:3, 1:4] = True + expected_pixels = pixels.copy() + expected_pixels[~expected_crop] = 0 + np.testing.assert_array_equal(cropping, expected_crop) + np.testing.assert_array_equal(image_payload_data(output), expected_pixels) + np.testing.assert_array_equal(image_payload_mask(output), expected_crop & parent_mask) + output_metadata = image_payload_metadata(output) + assert output_metadata.source_channel_axis == -1 + assert output_metadata.source_spatial_domain == domain + assert output_metadata.source_image_paths == ("/input/rgb.tif",) + assert measurements.row_count() == 1 + + +def test_prepared_threshold_keeps_owned_unit_interval_proof_and_mask(): + pixels = np.arange(20, dtype=np.uint8).reshape(4, 5) + mask = np.ones(pixels.shape, dtype=bool) + mask[0] = False + source = normalize_image_payload_intensity( + ImagePayloadMetadata.for_array(pixels).payload_with(pixels, mask), + dtype=np.float32, + ) + contract = _prepared_contract(threshold) + + projected = contract.raw_main_flow_call_argument(source) + + assert projected is source + assert image_payload_metadata(projected).unit_interval_intensity_scale == 255 + assert image_payload_mask(projected) is mask + np.testing.assert_array_equal(image_payload_data(projected), pixels.astype(np.float32) / 255) diff --git a/tests/unit/test_cellprofiler_function_contract_execution.py b/tests/unit/test_cellprofiler_function_contract_execution.py index 2a61995cec..76ca8ec07f 100644 --- a/tests/unit/test_cellprofiler_function_contract_execution.py +++ b/tests/unit/test_cellprofiler_function_contract_execution.py @@ -2,10 +2,16 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Sequence +from dataclasses import dataclass, replace +from functools import wraps +import inspect +from typing import Annotated, Any, Union import numpy as np import pytest +from arraybridge import ArrayPayload +import openhcs.core.callable_contract as callable_contract_module from openhcs.core.aligned_image_payload import ( AlignedImageStack, @@ -18,7 +24,12 @@ ImageArtifactType, MeasurementsArtifactType, ) -from openhcs.core.callable_contract import CallableContract, CallableMetadata +from openhcs.core.callable_contract import ( + CallableContract, + CallableMetadata, + CallableProjection, + callable_request, +) from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, RuntimePlaneAxisValueProjection, @@ -28,15 +39,26 @@ from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, + image_payload_mask, + image_payload_metadata, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.source_spatial_domain import SourceSpatialDomain from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( CellProfilerFunctionContractExecutor, ) +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter from openhcs.processing.backends.cellprofiler.alignment import AlignShiftMeasurement -from openhcs.processing.backends.cellprofiler.morphology import morphologicalskeleton +from openhcs.processing.backends.cellprofiler.morphology import ( + MorphOperation, + morph, + morphologicalskeleton, +) from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.processing.backends.lib_registry.unified_registry import ( + RuntimeCallablePolicy, RuntimeInvocationKwargPolicy, +) def _compiled_contract( @@ -55,7 +77,7 @@ def _compiled_contract( artifact_inputs=artifact_inputs, artifact_outputs=artifact_outputs, ), - ) + ).with_prepared_signature() def test_morphological_skeleton_executes_one_planar_runtime_image() -> None: @@ -76,6 +98,545 @@ def test_morphological_skeleton_executes_one_planar_runtime_image() -> None: assert image_payload_data(result).shape == image.shape +def test_registered_morph_distance_projects_raw_abi_and_retains_source_context() -> None: + contract = CallableContract.from_callable(morph) + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + artifact_outputs=(ArtifactSpec.output("Distance", ImageArtifactType),), + ), + ) + pixels = np.zeros((1, 7, 7), dtype=np.float32) + pixels[:, 2:5, 2:5] = 1 + mask = np.ones_like(pixels, dtype=bool) + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_names=("SavedLabels",), + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/tmp/saved-labels.tif",), + component_metadata=({"channel": "2", "well": "A01"},), + ), + ).payload_with(pixels, mask) + expected = contract.resolve_raw_runtime_callable()( + pixels[0], operation=MorphOperation.DISTANCE + ) + + result = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + {"operation": MorphOperation.DISTANCE}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ), + ) + + np.testing.assert_array_equal(image_payload_data(result), expected[None]) + np.testing.assert_array_equal(image_payload_mask(result), mask) + metadata = image_payload_metadata(result) + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_image_names == ("SavedLabels",) + assert metadata.source_image_provenance_planes.paths == ("/tmp/saved-labels.tif",) + assert metadata.source_image_provenance_planes.component_metadata == ( + {"channel": "2", "well": "A01"}, + ) + + +@pytest.mark.parametrize("payload_annotation", (False, True)) +def test_raw_slice_abi_retains_declared_payloads_and_projects_kwargs( + payload_annotation: bool, +) -> None: + seen = [] + + def array_raw(image: np.ndarray, *, labels: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + seen.append((image, labels)) + return image + labels + + def payload_raw(image: ArrayPayload, *, labels: np.ndarray) -> ArrayPayload: + assert isinstance(image, ArrayPayload) + seen.append((image, labels)) + return image_payload_metadata(image).payload_with( + image_payload_data(image) + labels, image_payload_mask(image), + ) + + raw = payload_raw if payload_annotation else array_raw + contract = _compiled_contract(raw, ProcessingContract.PURE_2D) + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4), dtype=np.float32), np.ones((3, 4), dtype=bool), + ) + selected_labels = np.full((3, 4), 5, dtype=np.float32) + executor = CellProfilerFunctionContractExecutor( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=2, + ), + ) + + assert contract.runtime_main_flow_call_argument(source) is source + result = executor.execute_pure_2d_slice( + contract, raw, source, + {"labels": RuntimeSliceAlignedValues(slices=(np.zeros((3, 4)), selected_labels)), + "adapter_only_control": 1}, + 1, 2, + ) + + assert seen[0][0] is (source if payload_annotation else image_payload_data(source)) + assert seen[0][1] is selected_labels + np.testing.assert_array_equal(image_payload_data(result), np.full((3, 4), 6)) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(source)) + assert image_payload_metadata(result).source_image_names == ("DNA",) + + +def test_raw_slice_abi_does_not_relax_output_mask_shape_validation() -> None: + def bad_shape(image: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + return image[:1] + + contract = _compiled_contract(bad_shape, ProcessingContract.PURE_2D) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4)), np.ones((3, 4), dtype=bool), + ) + executor = CellProfilerFunctionContractExecutor( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ), + ) + with pytest.raises(ValueError, match="[Mm]ask.*shape|shape.*[Mm]ask"): + executor.execute_pure_2d_slice(contract, bad_shape, source, {}, 0, 1) + + +@pytest.mark.parametrize("payload_annotation", (False, True)) +@pytest.mark.parametrize( + ("processing_contract", "execution_mode"), + ( + (ProcessingContract.PURE_2D, ImagePayloadExecutionMode.NATURAL), + (ProcessingContract.PURE_3D, ImagePayloadExecutionMode.NATURAL), + (ProcessingContract.PURE_2D, ImagePayloadExecutionMode.FULL_STACK), + (ProcessingContract.PURE_3D, ImagePayloadExecutionMode.FULL_STACK), + ), +) +def test_raw_abi_projection_matches_declared_argument_in_single_and_full_stack_modes( + payload_annotation: bool, + processing_contract: ProcessingContract, + execution_mode: ImagePayloadExecutionMode, +) -> None: + seen = [] + + def array_raw(image: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + seen.append(image) + return image + + def payload_raw(image: ArrayPayload) -> ArrayPayload: + assert isinstance(image, ArrayPayload) + seen.append(image) + return image + + raw = payload_raw if payload_annotation else array_raw + contract = _compiled_contract(raw, processing_contract) + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + artifact_outputs=(ArtifactSpec.output("Result", ImageArtifactType),), + ), + ) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4)), None, + ) + + result = CellProfilerFunctionContractExecutor().execute( + contract, raw, source, {}, execution_mode=execution_mode, + ) + + assert seen == [source if payload_annotation else image_payload_data(source)] + assert seen[0] is (source if payload_annotation else image_payload_data(source)) + assert result is seen[0] + + +@pytest.mark.parametrize( + ("annotation", "retains_payload"), + ( + (np.ndarray, False), + (ArrayPayload, True), + (RuntimeArrayData, True), + (Union[ArrayPayload, np.ndarray], True), + (Annotated[ArrayPayload, "source context"], True), + (Annotated[RuntimeArrayData, "source context"], True), + (Union[Annotated[ArrayPayload, "source context"], np.ndarray], True), + (Union[Annotated[RuntimeArrayData, "source context"], None], True), + (object, True), + (Any, False), + (inspect.Parameter.empty, False), + (Sequence[ArrayPayload], False), + (tuple[ArrayPayload, ...], False), + ), +) +def test_canonical_argument_projection_uses_declared_nominal_annotation( + annotation: object, retains_payload: bool, +) -> None: + def raw(image): + return image + + if annotation is not inspect.Parameter.empty: + raw.__annotations__["image"] = annotation + contract = _compiled_contract(raw, ProcessingContract.PURE_2D) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4)), None, + ) + expected = source if retains_payload else image_payload_data(source) + + assert contract.raw_main_flow_call_argument(source) is expected + assert contract.runtime_main_flow_call_argument(source) is expected + adapted = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + assert adapted.raw_main_flow_call_argument(source) is expected + assert adapted.runtime_main_flow_call_argument(source) is source + + +def test_raw_slice_argument_retains_buffer_identity_for_inplace_array_callable() -> None: + def inplace(image: np.ndarray) -> np.ndarray: + image[0, 0] = 7 + return image + + contract = _compiled_contract(inplace, ProcessingContract.PURE_2D) + pixels = np.zeros((3, 4)) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + pixels, None, + ) + executor = CellProfilerFunctionContractExecutor( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ), + ) + result = executor.execute_pure_2d_slice(contract, inplace, source, {}, 0, 1) + + assert image_payload_data(result) is pixels + assert pixels[0, 0] == 7 + + +def test_prepared_raw_abi_performs_no_introspection_until_explicit_refresh( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen = [] + + def raw(image: np.ndarray) -> np.ndarray: + seen.append(isinstance(image, ArrayPayload)) + return image_payload_data(image) + + contract = _compiled_contract( + raw, ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result", ImageArtifactType),), + ) + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, source_image_names=("DNA",), + ).payload_with(np.ones((12, 3, 4)), None) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=12, + ) + executor = CellProfilerFunctionContractExecutor(plane_projection=projection) + hints = [] + signatures = [] + original_hints = callable_contract_module.get_type_hints + original_signature = inspect.signature + + def counted_hints(func, **kwargs): + hints.append((func, kwargs)) + return original_hints(func, **kwargs) + + def counted_signature(func, **kwargs): + caller = inspect.currentframe().f_back + if caller.f_globals is callable_contract_module.__dict__: + signatures.append(caller.f_code.co_name) + return original_signature(func, **kwargs) + + monkeypatch.setattr(callable_contract_module, "get_type_hints", counted_hints) + monkeypatch.setattr(inspect, "signature", counted_signature) + + for annotation in (np.ndarray, ArrayPayload): + raw.__annotations__["image"] = annotation + contract = contract.with_prepared_signature() + result = executor.execute( + contract, raw, source, {}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=projection, + ) + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(source)) + + assert seen == [False] * 12 + [True] * 12 + assert hints == [(raw, {"include_extras": True})] * 2 + assert signatures == ["resolve_signature"] * 2 + assert not hasattr(executor, "_raw_argument_types") + + +def test_raw_abi_derivation_follows_existing_input_guards(monkeypatch) -> None: + def raw(image: np.ndarray) -> np.ndarray: + raise AssertionError("Invalid image reached raw execution") + + contract = _compiled_contract(raw, ProcessingContract.PURE_2D) + + def reject_premature_annotations(self): + raise AssertionError("Input guard resolved the raw ABI prematurely") + + monkeypatch.setattr( + CallableContract, "canonical_parameter_declarations", + property(reject_premature_annotations), + ) + with pytest.raises(TypeError, match="requires AlignedImageStack"): + CellProfilerFunctionContractExecutor().execute( + contract, raw, np.zeros((2, 3)), {}, + execution_mode=ImagePayloadExecutionMode.ALIGNED_MULTI_IMAGE_STACK, + ) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=2, + ) + mismatched_image = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + ).payload_with(np.zeros((2, 3, 4)), None) + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="conflicts"): + CellProfilerFunctionContractExecutor().execute( + contract, raw, mismatched_image, {}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=projection, + ) + + +def test_raw_abi_child_preserves_cooperative_executor_constructor_contract() -> None: + constructed = [] + raw_calls = [] + + class CooperativeExecutor(CellProfilerFunctionContractExecutor): + def __init__(self, plane_projection=None): + super().__init__(plane_projection=plane_projection) + constructed.append(self) + + def execute_pure_2d_slice(self, *args, **kwargs): + raw_calls.append(self) + return super().execute_pure_2d_slice(*args, **kwargs) + + def raw(image: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + return image + + contract = _compiled_contract( + raw, ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result", ImageArtifactType),), + ) + source = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE).payload_with( + np.ones((2, 3, 4)), None, + ) + executor = CooperativeExecutor() + result = executor.execute( + contract, raw, source, {}, execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=2, + ), + ) + + assert len(constructed) == 2 + assert raw_calls == [constructed[1]] * 2 + assert not hasattr(executor, "_raw_argument_types") + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(source)) + + def foreign_raw(image: ArrayPayload) -> ArrayPayload: + assert isinstance(image, ArrayPayload) + return image + + foreign_contract = _compiled_contract(foreign_raw, ProcessingContract.PURE_2D) + child = constructed[1] + policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) + assert policy.contract_invocation(foreign_contract, foreign_raw, source, {}).call() is source + assert not hasattr(child, "_scope_contract") + + +def test_actual_prepared_executor_planes_make_no_signature_or_hint_queries(monkeypatch) -> None: + calls = [] + def raw(image: np.ndarray, scale: int = 2) -> np.ndarray: + calls.append(image.shape) + return image * scale + CallableProjection.from_callable(raw).warm_canonical_signature() + contract = _compiled_contract( + raw, ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result",ImageArtifactType),), + ) + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE,source_image_names=("DNA",), + ).payload_with(np.ones((12,3,4)),None) + projection = RuntimePlaneAxisValueProjection.preserve(axis=RuntimePlaneAxis.RUNTIME_SLICE,axis_size=12) + def forbidden(*args, **kwargs): + raise AssertionError("Prepared execution introspected a live callable") + monkeypatch.setattr(inspect,"signature",forbidden) + monkeypatch.setattr(callable_contract_module,"get_type_hints",forbidden) + result = CellProfilerFunctionContractExecutor().execute( + contract,raw,source,{},execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=projection, + ) + np.testing.assert_array_equal(image_payload_data(result),np.full((12,3,4),2)) + assert calls == [(3,4)]*12 + + +def test_authored_distinct_raw_abi_batch_has_no_runtime_queries_and_filters_controls(monkeypatch) -> None: + from openhcs.core.processing_preparation import CallablePreparation + calls = [] + def raw(image: np.ndarray, scale: int = 2) -> np.ndarray: + calls.append(image.shape) + return image * scale + @wraps(raw) + def wrapper(*args, **kwargs): + return raw(*args, **kwargs) + wrapper.__signature__ = inspect.signature(raw).replace(parameters=[ + *inspect.signature(raw).parameters.values(), + inspect.Parameter("injected",inspect.Parameter.KEYWORD_ONLY,default=False), + ]) + monkeypatch.setattr(CallablePreparation,"prepare",lambda self:None) + contract = CallableContract.from_prepared_callable(wrapper) + contract = replace(contract,metadata=replace( + contract.metadata,processing_contract=ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result",ImageArtifactType),), + )) + source = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE,source_image_names=("DNA",)).payload_with(np.ones((12,3,4)),None) + projection = RuntimePlaneAxisValueProjection.preserve(axis=RuntimePlaneAxis.RUNTIME_SLICE,axis_size=12) + def forbidden(*args, **kwargs): + raise AssertionError("Prepared authored raw target queried a live signature") + monkeypatch.setattr(inspect,"signature",forbidden) + monkeypatch.setattr(callable_contract_module,"get_type_hints",forbidden) + result = CellProfilerFunctionContractExecutor().execute( + contract,wrapper,source,{"injected":True}, + execution_mode=ImagePayloadExecutionMode.NATURAL,plane_projection=projection, + ) + np.testing.assert_array_equal(image_payload_data(result),np.full((12,3,4),2)) + assert calls == [(3,4)]*12 + + +def test_compiled_slice_execution_resolves_raw_target_once_and_preserves_request_binding( + monkeypatch: pytest.MonkeyPatch, +) -> None: + @dataclass(frozen=True, slots=True) + class ScaleRequest: + image: object + scale: int + + seen: list[tuple[int, ...]] = [] + + @callable_request(ScaleRequest) + def request_bound(request: ScaleRequest) -> object: + data = image_payload_data(request.image) + seen.append(data.shape) + return data * request.scale + + @wraps(request_bound) + def decorated(*args: object, **kwargs: object) -> object: + raise AssertionError("CellProfiler invoked an outer runtime wrapper") + + contract = CallableContract.from_callable(decorated).with_prepared_signature() + contract = replace( + contract, + metadata=replace( + contract.metadata, + processing_contract=ProcessingContract.PURE_2D, + ), + ) + raw_target = contract.resolve_canonical_raw_callable() + resolve_raw = CallableContract.resolve_raw_runtime_callable + resolutions: list[CallableContract] = [] + + def resolve_once(self: CallableContract) -> Callable[..., object]: + resolutions.append(self) + return resolve_raw(self) + + def reject_reconstruction(*args: object, **kwargs: object) -> CallableContract: + raise AssertionError("Runtime execution rebuilt its compiled contract") + + monkeypatch.setattr(CallableContract, "resolve_raw_runtime_callable", resolve_once) + monkeypatch.setattr(CallableContract, "from_callable", reject_reconstruction) + image = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE + ).payload_with(np.ones((3, 4, 5), dtype=np.float32), None) + + result = CellProfilerFunctionContractExecutor().execute( + contract, + raw_target, + image, + {"scale": 4, "adapter_control": 17}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=3, + ), + ) + + assert resolutions == [contract] + assert seen == [(4, 5)] * 3 + assert isinstance(result, RuntimeSliceAlignedValues) + np.testing.assert_array_equal( + np.stack(tuple(image_payload_data(value) for value in result.slices)), + np.full((3, 4, 5), 4.0), + ) + + +@pytest.mark.parametrize("canonical_default", (False, True)) +def test_flexible_dispatch_preserves_canonical_signature_control_defaults( + canonical_default: bool, +) -> None: + seen: list[tuple[tuple[int, ...], bool]] = [] + + def raw(image: np.ndarray, *, slice_by_slice: bool = False) -> np.ndarray: + seen.append((image.shape, slice_by_slice)) + return image + + @wraps(raw) + def decorated(*args: object, **kwargs: object) -> object: + raise AssertionError("CellProfiler invoked an outer runtime wrapper") + + signature = inspect.signature(raw) + decorated.__signature__ = signature.replace( + parameters=( + parameter.replace(default=canonical_default) + if parameter.name == "slice_by_slice" + else parameter + for parameter in signature.parameters.values() + ) + ) + contract = _compiled_contract(decorated, ProcessingContract.FLEXIBLE) + image = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE + ).payload_with(np.ones((3, 4, 5), dtype=np.float32), None) + + CellProfilerFunctionContractExecutor().execute( + contract, + decorated, + image, + {}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=3, + ), + ) + + assert seen == ( + [((4, 5), True)] * 3 + if canonical_default + else [((3, 4, 5), False)] + ) + + @pytest.mark.parametrize("processing_contract", tuple(ProcessingContract)) @pytest.mark.parametrize( ("image_mode", "expected_call_count"), diff --git a/tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py b/tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py new file mode 100644 index 0000000000..933738e65e --- /dev/null +++ b/tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py @@ -0,0 +1,127 @@ +"""Geometry declarations preserve the payload their raw implementations consume.""" + +from dataclasses import replace +from typing import get_type_hints + +import numpy as np +import pytest + +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.callable_contract import CallableContract +from openhcs.core.runtime_array_values import RuntimeArrayData +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter +from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( + CellProfilerFunctionContractExecutor, +) +from openhcs.processing.backends.cellprofiler.image_geometry import ( + mask_image, + resize, + resize_volumetric, +) + + +def _prepared_contract(func): + contract = CallableContract.from_callable(func).with_prepared_signature() + return replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + + +@pytest.mark.parametrize("func", (resize, resize_volumetric)) +def test_prepared_resize_preserves_typed_primary_and_bare_array(func): + pixels = np.arange(24, dtype=np.float32).reshape(4, 6) + source = ImagePayloadMetadata( + source_path="/images/input.tif", source_image_names=("Original",), + ).payload_with(pixels, pixels % 2 == 0) + contract = _prepared_contract(func) + + assert contract.raw_main_flow_call_argument(source) is source + assert contract.raw_main_flow_call_argument(pixels) is pixels + + +@pytest.mark.parametrize("axis", (None, RuntimePlaneAxis.RUNTIME_SLICE)) +@pytest.mark.parametrize("z_factor", (1.0, 2.0)) +def test_prepared_full_stack_resize_preserves_existing_metadata_and_mask(axis, z_factor): + planes = tuple( + ImagePayloadMetadata( + source_path=f"/images/z{index}.tif", + source_image_names=("Original",), + source_component_metadata={"z_index": index, "timepoint": 1, "well": "W001"}, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(8, 10), + ), + ).payload_with(np.full((4, 6), index, dtype=np.float32), None) + for index in (3, 1, 2) + ) + metadata = ImagePayloadMetadata.compose(planes).replace_fields(plane_axis=axis) + pixels = np.stack(tuple(image_payload_data(plane) for plane in planes)) + mask = np.ones(pixels.shape, dtype=bool) + mask[:, 0, 0] = False + source = metadata.payload_with(pixels, mask) + contract = _prepared_contract(resize_volumetric) + canonical = contract.resolve_canonical_raw_callable() + kwargs = dict( + resizing_factor_x=0.5, resizing_factor_y=0.5, resizing_factor_z=z_factor, + ) + expected = canonical(source, **kwargs) + + result = CellProfilerFunctionContractExecutor().execute( + contract, canonical, source, kwargs, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(expected)) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(expected)) + assert image_payload_metadata(result) == image_payload_metadata(expected) + assert image_payload_metadata(result).plane_axis is axis + assert image_payload_metadata(result).source_image_paths == metadata.source_image_paths + assert image_payload_metadata(result).source_provenance == metadata.source_provenance + assert image_payload_metadata(result).source_spatial_domain.source_shape_yx == (2, 3) + assert image_payload_metadata(source) is metadata + np.testing.assert_array_equal(image_payload_data(source), pixels) + np.testing.assert_array_equal(image_payload_mask(source), mask) + + +@pytest.mark.parametrize("func", (resize, resize_volumetric)) +def test_prepared_resize_bare_pixels_preserve_canonical_return_contract(func): + pixels = np.arange(24, dtype=np.float32).reshape(4, 6) + contract = _prepared_contract(func) + canonical = contract.resolve_canonical_raw_callable() + kwargs = dict(resizing_factor_x=0.5, resizing_factor_y=0.5) + if func is resize_volumetric: + kwargs["resizing_factor_z"] = 1.0 + expected = canonical(pixels, **kwargs) + + result = CellProfilerFunctionContractExecutor().execute( + contract, canonical, pixels, kwargs, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + assert isinstance(result, RuntimeArrayData) + assert type(result) is type(expected) + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(expected)) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(expected)) + assert image_payload_metadata(result) == image_payload_metadata(expected) + + +def test_mask_image_declares_its_actual_nominal_return_family(): + contract = _prepared_contract(mask_image) + canonical = contract.resolve_canonical_raw_callable() + assert get_type_hints(canonical)["return"] == RuntimeArrayData + pixels = np.arange(12, dtype=np.float32).reshape(3, 4) + source = ImagePayloadMetadata(source_path="/images/input.tif").payload_with(pixels) + result = canonical(source, pixels > 3) + assert isinstance(result, RuntimeArrayData) + assert image_payload_metadata(result).source_image_paths == ("/images/input.tif",) diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index 9ba979906e..43240f69c8 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -10,6 +10,10 @@ import openhcs.processing.backends.cellprofiler.secondary as iso import openhcs.processing.backends.cellprofiler.secondary as ito +from openhcs.core.runtime_relationships import ( + DirectParentReferenceFeatureDeclaration, + DirectParentReferenceMeasurementFeature, +) from openhcs.constants.constants import ( AllComponents, Backend, @@ -50,6 +54,7 @@ from openhcs.core.callable_contract import ( CallableContract, CallableMetadata, + ImagePayloadConsumption, attach_callable_contract_metadata, runtime_image_execution_mode, ) @@ -251,8 +256,6 @@ CellProfilerOutputRecorder, ) from openhcs.interop.cellprofiler.runtime.relationship_measurement_rows import ( - DirectParentReferenceFeatureDeclaration, - DirectParentReferenceMeasurementFeature, RelationshipMeasurementRows, ) from openhcs.processing.backends.cellprofiler.alignment import ( @@ -1368,7 +1371,9 @@ def test_runtime_binding_uses_full_compiled_special_input_contract() -> None: np.testing.assert_array_equal(image_payload_data(topology_inputs[1]), 1.0) -def test_image_artifact_resolution_uses_declared_artifact_alias() -> None: +def test_image_artifact_resolution_uses_declared_artifact_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: spec = ArtifactSpec.input("IllumStain1", ImageArtifactType) source = ImagePayloadMetadata( source_image_provenance_planes=SourceImageProvenancePlanes.from_components( @@ -1394,6 +1399,17 @@ def test_image_artifact_resolution_uses_declared_artifact_alias() -> None: ).source_image_name(RuntimeArtifactInputRequest(spec=spec, value=source)) is None ) + strategy = RuntimeArtifactTypeStrategy.for_artifact_type(ImageArtifactType) + + def reject_second_resolution(*args, **kwargs): + raise AssertionError("Source-name projection resolved the image again") + + monkeypatch.setattr( + type(strategy), "raw_runtime_input_value", reject_second_resolution + ) + # The original name remains a contributor; the alias alone is not a + # unique represented source name. + assert strategy.source_image_name_from_value(payload) is None def test_main_flow_image_artifact_selects_declared_alias_plane() -> None: @@ -1590,7 +1606,9 @@ def test_main_flow_image_artifact_preserves_declared_runtime_slice_stack() -> No assert image_payload_metadata(payload).source_image_names == ("Stain1",) -def test_main_flow_projection_binds_singleton_broadcast_artifact_as_2d() -> None: +def test_main_flow_projection_binds_singleton_broadcast_artifact_as_2d( + monkeypatch: pytest.MonkeyPatch, +) -> None: original_spec = ArtifactSpec.input("OrigStain1", ImageArtifactType) illumination_spec = ArtifactSpec.input( "IllumStain1", @@ -1643,12 +1661,25 @@ def test_main_flow_projection_binds_singleton_broadcast_artifact_as_2d() -> None executor = _module_executor(contract) _activate_runtime_contract(executor.callable_contract, runtime) + strategy_type = type( + RuntimeArtifactTypeStrategy.for_artifact_type(ImageArtifactType) + ) + resolve_image = strategy_type.raw_runtime_input_value + resolved_specs: list[ArtifactSpec] = [] + + def record_resolution(self, request: RuntimeArtifactInputRequest): + resolved_specs.append(request.spec) + return resolve_image(self, request) + + monkeypatch.setattr(strategy_type, "raw_runtime_input_value", record_resolution) + image_request = executor._image_request( current_image, runtime, module_type=CorrectIlluminationApplyModule, active_input_specs=contract.artifact_inputs.specs, ) + assert resolved_specs == [original_spec] runtime_kwargs = executor._runtime_input_kwargs( runtime, current_image, @@ -1661,6 +1692,55 @@ def test_main_flow_projection_binds_singleton_broadcast_artifact_as_2d() -> None assert image_payload_data(runtime_kwargs["illumination_function"]).shape == (4, 5) +def test_composed_measurement_images_keep_declared_aliases_and_resolved_payload() -> None: + image_spec = ArtifactSpec.input("DeclaredImage", ImageArtifactType) + contract = _compiled_callable_contract( + CellProfilerModule.require_module("MeasureObjectIntensity").require_callable(), + artifact_inputs=(image_spec,), + ) + contract = replace( + contract, + metadata=replace( + contract.metadata, + image_payload_consumption=ImagePayloadConsumption.COMPOSED, + ), + ) + image = np.ones((4, 5), dtype=np.float32) + runtime = _FakeCellProfilerRuntime( + {image_spec.name: image}, + artifact_input_edges=(_artifact_input_edge_for_test(image_spec),), + ) + executor = _module_executor(contract) + _activate_runtime_contract(contract, runtime) + image_request = CellProfilerImageRequest( + payload=image, + source_image_name="RuntimeAlias", + source_aliases=("RuntimeAlias",), + image_count=1, + execution_mode=ImagePayloadExecutionMode.NATURAL, + ) + + (measurement_image,) = executor._measurement_image_inputs( + runtime, + image, + image_request, + module_type=MeasureObjectIntensityModule, + ) + + assert measurement_image.source_aliases == (image_spec.name,) + assert measurement_image.payload is image_request.payload + assert measurement_image.execution_mode is image_request.execution_mode + assert measurement_image.plane_projection is image_request.plane_projection + assert not measurement_image.align_to_labels + with pytest.raises(ValueError, match="requires a composed measurement image request"): + executor._measurement_image_inputs( + runtime, + image, + None, + module_type=MeasureObjectIntensityModule, + ) + + def test_repeated_broadcast_inputs_consume_their_declared_source_group_axes() -> None: original_specs = tuple( ArtifactSpec.input(f"OrigStain{index}", ImageArtifactType) for index in (1, 2) diff --git a/tests/unit/test_cellprofiler_prepared_payload_abi.py b/tests/unit/test_cellprofiler_prepared_payload_abi.py new file mode 100644 index 0000000000..bf0f122d11 --- /dev/null +++ b/tests/unit/test_cellprofiler_prepared_payload_abi.py @@ -0,0 +1,127 @@ +"""Compiled raw calls preserve metadata required by declared image consumers.""" + +from dataclasses import replace + +import numpy as np +from scipy import ndimage + +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.function_patterns import NormalizeFunctionGroupAuthority +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, + normalize_image_payload_intensity, +) +from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest +from openhcs.core.runtime_object_labels import object_label_dense_array +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter +from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( + CellProfilerFunctionContractExecutor, +) +from openhcs.processing.backends.cellprofiler.color import ColorToGrayMode, color_to_gray +from openhcs.processing.backends.cellprofiler.median_filter import medianfilter +from openhcs.processing.backends.cellprofiler.secondary import ( + SecondaryMethod, + identify_secondary_objects, +) + + +def _execute_prepared(func, source, kwargs, mode=ImagePayloadExecutionMode.NATURAL): + contract = NormalizeFunctionGroupAuthority().normalize("image", func).items[0].contract + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + assert contract.metadata.canonical_signature is not None + assert contract.raw_main_flow_call_argument(source) is source + return CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + kwargs, + execution_mode=mode, + ) + + +def test_prepared_color_to_gray_preserves_channel_mask_and_spatial_domain(): + pixels = np.arange(4 * 5 * 3, dtype=np.float32).reshape(4, 5, 3) + mask = np.ones(pixels.shape[:2], dtype=bool) + mask[0] = False + mask[-1] = False + domain = SourceSpatialDomain(origin_yx=(2, 3), source_shape_yx=(8, 10)) + source = ImagePayloadMetadata( + source_channel_axis=-1, + source_image_names=("RGB",), + source_path="/input/rgb.tif", + source_spatial_domain=domain, + ).payload_with(pixels, mask) + + result = _execute_prepared( + color_to_gray, source, + {"mode": ColorToGrayMode.COMBINE, + "channel_indices": (0, 2), "contributions": (1.0, 3.0)}, + ) + + np.testing.assert_array_equal(image_payload_data(result), (pixels[..., 0] + 3 * pixels[..., 2]) / 4) + np.testing.assert_array_equal(image_payload_mask(result), mask) + metadata = image_payload_metadata(result) + assert metadata.source_channel_axis is None + assert metadata.source_spatial_domain == domain + assert metadata.source_image_paths == ("/input/rgb.tif",) + + +def test_prepared_medianfilter_keeps_original_intensity_scale_and_mask(): + pixels = np.arange(25, dtype=np.uint16).reshape(5, 5) + mask = np.ones(pixels.shape, dtype=bool) + mask[0] = False + source = normalize_image_payload_intensity( + ImagePayloadMetadata.for_array(pixels).payload_with(pixels, mask), + dtype=np.float32, + ) + + result = _execute_prepared( + medianfilter, source, {"window_size": 3}, ImagePayloadExecutionMode.FULL_STACK, + ) + + np.testing.assert_array_equal( + image_payload_data(result), + ndimage.median_filter(image_payload_data(source), size=3, mode="constant"), + ) + np.testing.assert_array_equal(image_payload_mask(result), mask) + assert image_payload_metadata(result).unit_interval_intensity_scale == 65535 + assert image_payload_data(result).dtype == np.float32 + + +def test_prepared_secondary_object_output_keeps_actual_source_spatial_domain(): + pixels = np.zeros((9, 9), dtype=np.float32) + domain = SourceSpatialDomain(origin_yx=(3, 4), source_shape_yx=(15, 16)) + source = ImagePayloadMetadata( + source_image_names=("DNA",), source_path="/input/dna.tif", + source_spatial_domain=domain, + ).payload_with(pixels, None) + seed_pixels = np.zeros(pixels.shape, dtype=np.int32) + seed_pixels[4, 4] = 1 + seeds = SourceImageObjectLabelBuildRequest( + image=source, labels=seed_pixels, declared_object_ids=(1,), + ).payload() + + result = _execute_prepared( + identify_secondary_objects, source, + {"primary_labels": seeds, "method": SecondaryMethod.DISTANCE_N, + "distance_to_dilate": 1, "fill_holes": False}, + ) + + _image, _rows, relationship, objects = result + expected = ndimage.distance_transform_edt(seed_pixels == 0) <= 1 + np.testing.assert_array_equal(object_label_dense_array(objects), expected.astype(np.int32)) + assert objects.source_spatial_domain.origin_yx == domain.origin_yx + assert objects.source_spatial_domain.source_shape_yx == domain.source_shape_yx + assert objects.source_provenance == image_payload_metadata(source).source_provenance + assert relationship.source_ids == (1,) + assert relationship.target_ids == (1,) diff --git a/tests/unit/test_cellprofiler_relationship_contracts.py b/tests/unit/test_cellprofiler_relationship_contracts.py index 74ee1d575a..64a352ebb2 100644 --- a/tests/unit/test_cellprofiler_relationship_contracts.py +++ b/tests/unit/test_cellprofiler_relationship_contracts.py @@ -6,6 +6,10 @@ import numpy as np +from openhcs.core.runtime_relationships import ( + DirectParentReferenceFeatureDeclaration, + DirectParentReferenceMeasurementFeature, +) from openhcs.core.artifacts import ( ArtifactInputPlan, ArtifactOutputPlan, @@ -49,10 +53,6 @@ ) from openhcs.interop.cellprofiler.pipeline_import import import_cellprofiler_pipeline from openhcs.interop.cellprofiler.setting_names import setting_values -from openhcs.interop.cellprofiler.runtime.relationship_measurement_rows import ( - DirectParentReferenceFeatureDeclaration, - DirectParentReferenceMeasurementFeature, -) from openhcs.processing.backends.cellprofiler.neighbors import ( MeasureObjectNeighborsModule, ) diff --git a/tests/unit/test_dense_object_coordinate_reduction.py b/tests/unit/test_dense_object_coordinate_reduction.py new file mode 100644 index 0000000000..a136d5828c --- /dev/null +++ b/tests/unit/test_dense_object_coordinate_reduction.py @@ -0,0 +1,298 @@ +"""Storage reduction preserves its generic oracle and compiled backend domains.""" + +import os +import subprocess +import sys +import textwrap +import warnings +from pathlib import Path + +import numpy as np +import pytest + +from openhcs.core.runtime_object_labels import ( + ObjectLabelPayload, + ObjectLabelStorageStrategy, + ObjectLabelVariantData, + object_label_axis_centers, +) +from openhcs.core.runtime_sparse_labels import SparseIJVLabelRows + + +def generic_centers(labels, domain): + return ObjectLabelStorageStrategy.axis_centers( + ObjectLabelStorageStrategy.for_value(labels), labels, domain=domain + ) + + +def assert_centers(actual, expected): + for actual_array, expected_array in zip( + (*actual[0], actual[1]), (*expected[0], expected[1]), strict=True + ): + np.testing.assert_array_equal(actual_array, expected_array) + assert actual_array.tobytes() == expected_array.tobytes() + assert actual_array.dtype == expected_array.dtype + assert actual_array.strides == expected_array.strides + assert actual_array.flags.owndata == expected_array.flags.owndata + assert actual_array.base is None + assert not np.shares_memory(*actual[0][:2]) + + +@pytest.mark.parametrize("wrapped", (False, True)) +@pytest.mark.parametrize("shape", ((4, 5), (3, 4, 5))) +def test_dense_storage_reduces_without_sparse_materialization(monkeypatch, wrapped, shape): + labels = np.zeros(shape, dtype=np.int32) + labels[..., 1:3, 2:4] = 1 + labels[..., 3, 0] = 4 + domain = (1, 2, 4, 7) + expected = generic_centers(labels, domain) + value = ( + ObjectLabelPayload(variant_data=ObjectLabelVariantData(labels=labels)) + if wrapped else labels + ) + before = labels.copy() + + def reject_materialization(_cls, _labels): + raise AssertionError("Coordinate reduction materialized sparse pixels") + + monkeypatch.setattr(SparseIJVLabelRows, "from_dense_stack", classmethod(reject_materialization)) + assert_centers(object_label_axis_centers(value, domain=domain), expected) + np.testing.assert_array_equal(labels, before) + + +@pytest.mark.parametrize("layout", ("c", "f", "view", "readonly", "empty")) +def test_layout_and_mutable_pixels_preserve_generic_result(layout): + labels = np.arange(30, dtype=np.int32).reshape(5, 6) % 5 + if layout == "f": + labels = np.asfortranarray(labels) + elif layout == "view": + labels = labels[::-1, ::2] + elif layout == "readonly": + labels.flags.writeable = False + elif layout == "empty": + labels = np.zeros((2, 0, 3), dtype=np.int32) + domain = (1, 2, 4, 9) + assert_centers(object_label_axis_centers(labels, domain=domain), generic_centers(labels, domain)) + if labels.size and labels.flags.writeable: + labels.flat[0] = 4 + assert_centers(object_label_axis_centers(labels, domain=domain), generic_centers(labels, domain)) + + +@pytest.mark.parametrize("dtype", (np.bool_, np.float64, np.int64, np.uint64)) +def test_other_dtype_conversion_and_warning_policy_stays_generic(dtype): + labels = np.array([[0, 1], [2, 3]], dtype=dtype) + if dtype is np.float64: + labels[0] = (0.5, np.nan) + elif dtype is np.uint64: + labels[0, 1] = 2**32 + 1 + outcomes = [] + for call in (generic_centers, lambda x, domain: object_label_axis_centers(x, domain=domain)): + with warnings.catch_warnings(record=True) as observed: + warnings.simplefilter("always") + try: + result = call(labels, (1, 2, 3, 8)) + except ValueError as error: + result = (type(error), str(error)) + outcomes.append((result, [(type(w.message), str(w.message)) for w in observed])) + if dtype is np.uint64: + assert outcomes[0][0] == outcomes[1][0] + else: + assert_centers(outcomes[0][0], outcomes[1][0]) + assert outcomes[0][1] == outcomes[1][1] + + +def test_sparse_overlap_and_subtype_keep_existing_representation(): + sparse = SparseIJVLabelRows(np.array([[0, 0, 1], [0, 0, 2], [2, 3, 1]], dtype=np.int32)) + assert_centers(object_label_axis_centers(sparse, domain=(1, 2, 4)), generic_centers(sparse, (1, 2, 4))) + + class DenseSubtype(np.ndarray): + @property + def dtype(self): + raise AssertionError("Subtype properties must not be probed before generic coercion") + + subtype = np.array([[1, 0], [2, 2]], dtype=np.int32).view(DenseSubtype) + assert_centers(object_label_axis_centers(subtype, domain=(1, 2, 4)), generic_centers(subtype, (1, 2, 4))) + + +@pytest.mark.parametrize("shape", ((), (4,), (2, 3, 4, 5))) +def test_invalid_shape_keeps_original_error(shape): + labels = np.zeros(shape, dtype=np.int32) + with pytest.raises(ValueError) as original: + generic_centers(labels, (1,)) + with pytest.raises(type(original.value), match="SparseIJVLabelRows.from_dense_stack") as current: + object_label_axis_centers(labels, domain=(1,)) + assert str(current.value) == str(original.value) + + +@pytest.mark.parametrize("failure", (None, "iterate", 1, 2, 3, 4, 5)) +def test_domain_callbacks_follow_pixel_snapshot_and_original_allocation_order(failure): + outcomes = [] + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + labels = np.array([[1, 1], [2, 0]], dtype=np.int32) + effects = [] + + class DomainInteger(int): + def __gt__(self, other): + effects.append("compare") + return super().__gt__(other) + + def __add__(self, other): + effects.append("add") + result = super().__add__(other) + return float(result) if effects.count("add") == failure else result + + class MutatingDomain: + def __iter__(self): + effects.append("iterate") + labels[:] = 3 + labels.shape = (1, 2, 2) + if failure == "iterate": + raise RuntimeError("Domain iteration failed after changing pixels") + yield DomainInteger(4) + + try: + result = call(labels, MutatingDomain()) + except (RuntimeError, TypeError) as error: + outcome = (type(error), str(error)) + else: + outcome = result + outcomes.append((outcome, effects, labels.copy())) + + if failure is None: + assert_centers(outcomes[0][0], outcomes[1][0]) + np.testing.assert_array_equal(outcomes[1][0][1], (0, 2, 1, 0, 0)) + assert len(outcomes[1][0][0]) == 2 + else: + assert outcomes[0][0] == outcomes[1][0] + assert outcomes[0][1] == outcomes[1][1] + np.testing.assert_array_equal(outcomes[0][2], outcomes[1][2]) + + +@pytest.mark.parametrize("domain", ((4.0,), (np.int64(np.iinfo(np.int64).max),), (2**63,))) +def test_domain_minlength_errors_and_warnings_remain_numpy_owned(domain): + outcomes = [] + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + with warnings.catch_warnings(record=True) as observed: + warnings.simplefilter("always") + try: + call(np.array([[1, 0], [2, 1]], dtype=np.int32), domain) + except (TypeError, ValueError, OverflowError) as error: + outcome = (type(error), str(error)) + else: + raise AssertionError("Invalid minlength unexpectedly succeeded") + outcomes.append((outcome, [(type(w.message), str(w.message)) for w in observed])) + assert outcomes[0] == outcomes[1] + + +def test_invalid_geometry_precedes_domain_iteration(): + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + effects = [] + + class Domain: + def __iter__(self): + effects.append("iterate") + yield 1 + + with pytest.raises(ValueError, match="SparseIJVLabelRows.from_dense_stack"): + call(np.zeros((2, 3, 4, 5), dtype=np.int32), Domain()) + assert not effects + + +@pytest.mark.parametrize("shape,label_id", (((1, 1), 2**31 - 1), ((64, 64), 4096))) +def test_sparse_high_ids_reach_domain_error_before_moment_allocation(monkeypatch, shape, label_id): + from openhcs.core import runtime_object_labels + + def reject_moment_allocation(*_args): + raise AssertionError("Sparse ID span must not allocate a dense moment table") + + monkeypatch.setattr(runtime_object_labels, "_dense_label_coordinate_moments_numba", reject_moment_allocation) + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + labels = np.zeros(shape, dtype=np.int32) + labels.flat[0] = label_id + effects = [] + + class Domain: + def __iter__(self): + effects.append("iterate") + labels.flat[0] = 1 + raise RuntimeError("Domain rejected sparse high IDs") + + with pytest.raises(RuntimeError, match="Domain rejected sparse high IDs"): + call(labels, Domain()) + assert effects == ["iterate"] + assert labels.flat[0] == 1 + + +@pytest.mark.parametrize("dimensions", (2, 3)) +@pytest.mark.parametrize("layout", ("fortran", "reversed", "broadcast")) +def test_coordinate_scan_borrows_strided_input_without_flatten_copy(monkeypatch, dimensions, layout): + from openhcs.core import runtime_object_labels + + labels = np.array([[0, 1, 1], [2, 0, 2]], dtype=np.int32) + if dimensions == 3: + labels = np.stack((labels, labels)) + if layout == "fortran": + labels = np.asfortranarray(labels) + elif layout == "reversed": + labels = labels[..., ::-1] + else: + labels = np.broadcast_to(labels[..., :1, :], (*labels.shape[:-2], 4, 3)) + expected = generic_centers(labels, (1, 2, 4)) + before = labels.copy() + original = runtime_object_labels._dense_label_coordinate_moments_numba + requests = [] + + def borrowed_scan(coordinate_planes, maximum_label): + assert coordinate_planes.ndim == 3 + assert np.shares_memory(coordinate_planes, labels) + assert coordinate_planes.flags.writeable == labels.flags.writeable + assert coordinate_planes.shape == ((1, *labels.shape) if dimensions == 2 else labels.shape) + requests.append(coordinate_planes) + return original(coordinate_planes, maximum_label) + + monkeypatch.setattr(runtime_object_labels, "_dense_label_coordinate_moments_numba", borrowed_scan) + assert_centers(object_label_axis_centers(labels, domain=(1, 2, 4)), expected) + assert len(requests) == 1 + np.testing.assert_array_equal(labels, before) + + +def test_primary_objects_preparation_readies_all_admitted_storage_signatures(tmp_path): + script = textwrap.dedent(""" + from unittest.mock import patch + import numpy as np + from openhcs.core.callable_contract import prepare_processing_callable + from openhcs.core.runtime_object_labels import ( + _dense_label_coordinate_moments_numba, dense_label_centers_2d_numba, + object_label_axis_centers, + ) + from openhcs.processing.backends.cellprofiler.primary_objects import identify_primary_objects + assert not _dense_label_coordinate_moments_numba.signatures + prepare_processing_callable(identify_primary_objects) + kernels = (_dense_label_coordinate_moments_numba, dense_label_centers_2d_numba) + signatures = tuple(tuple(kernel.signatures) for kernel in kernels) + assert all(signatures) + + def reject_compilation(*arguments): + raise AssertionError(('Late coordinate preparation', tuple(map(str, arguments)))) + + from contextlib import ExitStack + with ExitStack() as guards: + for kernel in kernels: + guards.enter_context(patch.object(kernel, 'compile', side_effect=reject_compilation)) + guards.enter_context(patch.object(kernel._cache, 'load_overload', side_effect=reject_compilation)) + for shape in ((4, 5), (3, 4, 5)): + base = np.arange(np.prod(shape), dtype=np.int32).reshape(shape) % 4 + for labels in (base, np.asfortranarray(base), base[..., ::-1], + np.broadcast_to(base[..., :1, :], shape), base.copy()): + object_label_axis_centers(labels, domain=(1, 2, 3, 6)) + labels.flags.writeable = False + object_label_axis_centers(labels, domain=(1, 2, 3, 6)) + assert tuple(tuple(kernel.signatures) for kernel in kernels) == signatures + """) + environment = os.environ.copy() + environment.update(OPENHCS_CPU_ONLY="true", NUMBA_CACHE_DIR=str(tmp_path / "kernels")) + result = subprocess.run( + (sys.executable, "-c", script), cwd=Path(__file__).parents[2], + env=environment, capture_output=True, text=True, timeout=120, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/unit/test_derived_role_publication_435.py b/tests/unit/test_derived_role_publication_435.py new file mode 100644 index 0000000000..4a0155c0f1 --- /dev/null +++ b/tests/unit/test_derived_role_publication_435.py @@ -0,0 +1,160 @@ +"""Issue435: real saved image occurrences projected through two existing owners.""" + +from dataclasses import asdict, replace +import json + +import pytest +import numpy as np +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.memory import MemoryStorageBackend + +from test_function_outputs import ( + context_stub, function_step_plan, record_output_path, +) +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +from openhcs.constants.constants import Backend, Microscope +from openhcs.core.aligned_image_payload import AlignedImageSliceContext +from openhcs.core.artifacts import ArtifactOutputPlan, ImageArtifactType, ObjectLabelsArtifactType +from openhcs.core.compiled_step_plan import RuntimeArtifactMaterializationPlan +from openhcs.core.runtime_artifact_values import RuntimeValue +from openhcs.core.runtime_image_values import ImageMetadataPayload, ImagePayloadMetadata +from openhcs.core.runtime_stores import RuntimeValueStore +from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection +from openhcs.core.steps.function_outputs import ( + PrimaryImageMetadataTarget, RuntimeArtifactMaterializationAuthority, +) +from openhcs.processing.materialization import ( + ImageFileOptions, MaterializationSpec, MaterializedFilenameIdentity, +) + + +@pytest.mark.parametrize("scenario", ("same_occurrence", "different_path", "stale_scope", "conflicting_metadata", "different_kind")) +def test_saved_roles_publish_once_per_persisted_occurrence(tmp_path, scenario): + same_persisted_path = scenario != "different_path" + filemanager = FileManager({ + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + }) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + context.microscope_handler.microscope_type = Microscope.OPENHCS.value + context.runtime_value_store = RuntimeValueStore() + context.metadata_cache = {} + context.tiff_config = None + plan = function_step_plan("Derived role images") + plan.streaming_configs = {} + plan.write_backend = Backend.DISK.value + plan.output_plate_root = str(tmp_path) + plan.output_dir = tmp_path / "images" + plan.sub_dir = "images" + plan.analysis_results_dir = str(plan.output_dir) + plan.runtime_artifact_materialization = RuntimeArtifactMaterializationPlan( + persistent_enabled=True, persistent_backend=Backend.DISK.value, + ) + context.step_plans = {plan.step_index: plan} + filemanager.ensure_directory(plan.output_dir, Backend.MEMORY.value) + components = { + "well": "A01", "site": "1", "channel": "2", + "z_index": "1", "timepoint": "1", "extension": ".tif", + } + for role, scale in (("RawRole", 1.0), ("CappedRole", 0.5)): + filename = f"A01_s001_w2_z001_t001_{role}.tif" + output_plan = ArtifactOutputPlan( + name=role, path=f"/memory/A01_s001_w2_z001_t001_{role}.pkl", + artifact_type=(ObjectLabelsArtifactType if scenario == "different_kind" and role == "CappedRole" else ImageArtifactType), + producer_step_index=plan.step_index, + producer_step_scope_id=("different-step-scope" if scenario == "stale_scope" and role == "CappedRole" else plan.step_scope_id), + producer_step_name=plan.step_name, + materialization=MaterializationSpec(ImageFileOptions( + filename_suffix=".tif", + filename_identity=MaterializedFilenameIdentity.SOURCE_IDENTITY, + relative_path_template=(filename if same_persisted_path else filename.replace(".tif", "_independent.tif")), + )), + ) + plan.artifact_outputs[output_plan.ref()] = output_plan + payload = ImageMetadataPayload( + np.ones((4, 5), dtype=np.float32) * scale, + ImagePayloadMetadata( + source_component_metadata=components, + source_image_names=(role,), + source_voxel_spacing=SourceVoxelSpacing((0.65, 0.65)), + ), + ) + artifact_payload = payload + if role == "CappedRole" and scenario == "conflicting_metadata": + artifact_payload = payload.metadata.replace_fields(source_image_names=("IndependentRole",)).payload_with(payload.data) + if role == "CappedRole" and scenario == "different_kind": + artifact_payload = SourceImageObjectLabelBuildRequest( + image=payload, labels=np.ones(payload.data.shape, dtype=np.int32), + declared_object_count=1, declared_object_ids=(1,), + ).payload() + context.runtime_value_store.record( + RuntimeValue.normalize(output_plan, artifact_payload, axis_id="A01"), + path=output_plan.path, backend=Backend.MEMORY.value, + ) + destination = plan.output_dir / filename + filemanager.save(payload, str(destination), Backend.MEMORY.value) + filemanager.ensure_directory(plan.output_dir, Backend.DISK.value) + filemanager.save(payload.data, str(destination), Backend.DISK.value) + record_output_path( + context, plan, str(destination), image_metadata=payload.metadata, + output_context=AlignedImageSliceContext.main_flow( + output_key=role, artifact_kind=ImageArtifactType.value, + ), + ) + saved = RuntimeArtifactMaterializationAuthority.materialize(context, plan) + assert len(saved) == 2 + target = replace(PrimaryImageMetadataTarget.from_plan(plan), artifact_materializations=saved) + records = target.produced_records(context, plan) + artifacts = target.runtime_artifact_projection_paths(context, plan) + receipt = { + "source_scope": plan.step_scope_id, + "produced": [{"path": str(record.path_under(plan.output_dir)), + "identity": asdict(record.producer_identity), + "address": record.filename_address.as_component_metadata()} for record in records], + "materialized": [{"path": output.path, + "role": item.materialization.output_plan.name, + "producer_scope": item.materialization.output_plan.producer_step_scope_id, + "stored_key": repr(item.materialization.record.key)} + for item in saved for output in item.outputs_for_backend(Backend.DISK.value)], + "artifact_projections": [{"path": path, + "role": projection.projection_role.value, + "address": projection.address.as_component_metadata(), + "alias": projection.source_alias, + "kind": projection.artifact_kind.value} + for projection, path in artifacts], + } + (tmp_path / "persisted-occurrence-witness.json").write_text(json.dumps(receipt, indent=2)) + produced_paths = {item["path"] for item in receipt["produced"]} + materialized_paths = {item["path"] for item in receipt["materialized"]} + if same_persisted_path: + assert produced_paths == materialized_paths + else: + assert produced_paths.isdisjoint(materialized_paths) + expected_sums = {20.0} if scenario == "different_kind" else {20.0, 10.0} + assert {float(tifffile.imread(item["path"]).sum()) for item in receipt["materialized"]} == expected_sums + if scenario != "same_occurrence": + message = "Conflicting metadata for persisted image" if scenario == "conflicting_metadata" else "Duplicate source projection address" + with pytest.raises(ValueError, match=message) as rejection: + target.produced_projection_metadata(context, plan) + receipt["rejection"] = str(rejection.value) + (tmp_path / "persisted-occurrence-witness.json").write_text(json.dumps(receipt, indent=2)) + return + structured = target.produced_projection_metadata(context, plan) + assert len(structured["source_projection"]) == 2 + assert {item["source_alias"] for item in structured["source_projection"]} == {"RawRole", "CappedRole"} + target.write(context, produced_plan=plan) + document = json.loads((tmp_path / "openhcs_metadata.json").read_text()) + reopened = VirtualWorkspaceSourceProjection.from_openhcs_metadata(tmp_path, document) + projections = reopened.source_projections_by_virtual_path + relative_paths = {item["path"].removeprefix(str(tmp_path) + "/") for item in receipt["produced"]} + assert set(projections) == relative_paths | produced_paths + assert {projection.source_alias for projection in projections.values()} == {"RawRole", "CappedRole"} + for relative_path in relative_paths: + projection = projections[relative_path] + assert projection.address.as_component_metadata() == {key: components[key] for key in ("well", "site", "channel", "z_index", "timepoint")} + expected_scale = 1.0 if projection.source_alias == "RawRole" else 0.5 + np.testing.assert_array_equal(tifffile.imread(tmp_path / relative_path), np.ones((4, 5), dtype=np.float32) * expected_scale) diff --git a/tests/unit/test_exported_relationship_equivalence.py b/tests/unit/test_exported_relationship_equivalence.py new file mode 100644 index 0000000000..6eeba55b94 --- /dev/null +++ b/tests/unit/test_exported_relationship_equivalence.py @@ -0,0 +1,413 @@ +"""Saved directed edges must retain the native object-row correlations.""" + +from pathlib import Path + +import pytest + +from openhcs.core.equivalence.outputs import RuntimeOutputSnapshot +from openhcs.core.runtime_equivalence import ( + RuntimeMeasurementSnapshot, + runtime_measurement_equivalence, +) + +_HEADER = ( + "relationship_type,source_role,target_role,source_object,target_object," + "producer_module_number,parent_id,child_id,image_number,slice_count\n" +) + + +def _saved_outputs( + root: Path, *, edges: bool, swapped: bool = False +) -> RuntimeOutputSnapshot: + root.mkdir() + (root / "Cells.csv").write_text( + "ImageNumber,ObjectNumber,Children_Nuclei_Count\n" + "1,1,1\n1,2,1\n1,3,0\n2,1,1\n2,2,1\n2,3,0\n" + ) + parents = (2, 1) if swapped else (1, 2) + (root / "Nuclei.csv").write_text( + "ImageNumber,ObjectNumber,Parent_Cells\n" + + "".join( + f"{image},{child},{parents[child - 1]}\n" + for image in (1, 2) + for child in (1, 2) + ) + ) + if edges: + (root / "Relationships.csv").write_text( + _HEADER + + "".join( + f"{kind},{source_role},{target_role},{source},{target},11," + f"{parents[child - 1]},{child},{image},2\n" + for kind, source_role, target_role, source, target in ( + ("Parent", "parent", "child", "Cells", "Nuclei"), + ("Child", "child", "parent", "Nuclei", "Cells"), + ) + for image in (1, 2) + for child in (1, 2) + ) + ) + return RuntimeOutputSnapshot.from_output_root(root) + + +def test_saved_relationship_edges_join_native_parent_and_zero_child_counts( + tmp_path: Path, +) -> None: + reference = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "native", edges=False) + ) + candidate = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "openhcs", edges=True) + ) + assert runtime_measurement_equivalence(reference, candidate).is_equivalent + + +def test_relationship_endpoint_permutation_cannot_hide_in_equal_value_histograms( + tmp_path: Path, +) -> None: + reference = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "native", edges=False) + ) + candidate = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "openhcs", edges=True, swapped=True) + ) + assert not runtime_measurement_equivalence(reference, candidate).is_equivalent + + +def test_relationship_rows_must_agree_with_explicit_parent_measurements( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "openhcs", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.rows = tuple( + tuple( + "2" if index == 6 and row[7] == "1" and row[8] == "1" else value + for index, value in enumerate(row) + ) + for row in table.rows + ) + with pytest.raises(ValueError, match="relationship|Relationship"): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_ordinary_image_row_conflicts_remain_rejected(tmp_path: Path) -> None: + root = tmp_path / "ordinary" + root.mkdir() + (root / "Image.csv").write_text("ImageNumber,Count_Nuclei\n1,2\n1,3\n") + with pytest.raises(ValueError, match="conflicting observed values"): + RuntimeMeasurementSnapshot.from_output_snapshot( + RuntimeOutputSnapshot.from_output_root(root) + ) + + +@pytest.mark.parametrize( + "mutation, message", + [ + ( + lambda header, rows: (header, rows[:-1]), + "Relationship edges disagree|reverse declarations", + ), + (lambda header, rows: (header, (*rows, rows[0])), "duplicate directed edges"), + ( + lambda header, rows: ( + header, + tuple( + tuple( + "12" if i == 5 and row[0] == "Child" else value + for i, value in enumerate(row) + ) + for row in rows + ), + ), + "reverse declarations", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple( + "neighbor_source" if i == 1 else value + for i, value in enumerate(row) + ) + for row in rows + ), + ), + "Unsupported exported relationship roles", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple("999" if i == 7 else value for i, value in enumerate(row)) + for row in rows + ), + ), + "absent child", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple("2.5" if i == 6 else value for i, value in enumerate(row)) + for row in rows + ), + ), + "nonnegative integers", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple("2.5" if i == 9 else value for i, value in enumerate(row)) + for row in rows + ), + ), + "slice count must be an integer", + ), + ( + lambda header, rows: (header[1:], tuple(row[1:] for row in rows)), + "incomplete declaration", + ), + (lambda header, rows: (header, ()), "no determining declaration"), + ], +) +def test_saved_edge_schema_and_coverage_rejects_malformed_inputs( + tmp_path: Path, mutation, message: str +) -> None: + snapshot = _saved_outputs(tmp_path / "openhcs", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.header, table.rows = mutation(table.header, table.rows) + with pytest.raises((ValueError, TypeError), match=message): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_saved_relationship_counts_include_noncontiguous_zero_child_parents( + tmp_path: Path, +) -> None: + native = _saved_outputs(tmp_path / "native", edges=False) + candidate = _saved_outputs(tmp_path / "openhcs", edges=True) + for snapshot in (native, candidate): + for table in snapshot.tables: + table.rows = tuple( + tuple( + ( + "30" + if table.path.stem == "Cells" and i == 1 and value == "3" + else value + ) + for i, value in enumerate(row) + ) + for row in table.rows + ) + a = RuntimeMeasurementSnapshot.from_output_snapshot(native) + b = RuntimeMeasurementSnapshot.from_output_snapshot(candidate) + assert ( + a.correlated_relationships is not None + and b.correlated_relationships is not None + ) + assert runtime_measurement_equivalence(a, b).is_equivalent + + +def test_snapshot_cache_retains_known_correlations_and_unknown_typed_scope( + tmp_path: Path, +) -> None: + known = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "saved", edges=True) + ) + unknown = RuntimeMeasurementSnapshot(known.measurement_fact_counts) + for snapshot in (known, unknown): + restored = RuntimeMeasurementSnapshot.from_cache_payload( + snapshot.to_cache_payload() + ) + assert restored.measurement_fact_counts == snapshot.measurement_fact_counts + assert restored.correlated_relationships == snapshot.correlated_relationships + assert unknown.correlated_relationships is None + assert known.correlated_relationships is not None + + +def test_cached_relationship_preserves_declared_slice_count_and_endpoint_axes() -> None: + from openhcs.core.equivalence.keys import ( + RuntimeMeasurementFeatureKey, + RuntimeMeasurementSubjectKey, + ) + from openhcs.core.runtime_measurements import MeasurementScope + from openhcs.core.runtime_relationships import ( + ObjectInstanceKey, + ObjectInstanceRelationship, + ) + + key = RuntimeMeasurementFeatureKey( + RuntimeMeasurementSubjectKey(MeasurementScope.RELATIONSHIP, "child"), "parent" + ) + graph = ObjectInstanceRelationship( + (ObjectInstanceKey(2, 0),), (ObjectInstanceKey(5, 1),), 2 + ) + snapshot = RuntimeMeasurementSnapshot({}, {key: graph}) + assert ( + RuntimeMeasurementSnapshot.from_cache_payload( + snapshot.to_cache_payload() + ).correlated_relationships + == snapshot.correlated_relationships + ) + + +def test_saved_relationship_omitted_slice_count_preserves_optional_payload_contract( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "saved", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.header = table.header[:-1] + table.rows = tuple(row[:-1] for row in table.rows) + result = RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + assert result.correlated_relationships is not None + + +def test_saved_relationship_row_order_does_not_change_directed_correlation( + tmp_path: Path, +) -> None: + native = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "native", edges=False) + ) + snapshot = _saved_outputs(tmp_path / "saved", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.rows = table.rows[:4] + table.rows[4:][::-1] + result = RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + assert runtime_measurement_equivalence(native, result).is_equivalent + + +def test_unparented_zero_edges_still_validate_actual_child_domain( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "saved", edges=True) + for table in snapshot.tables: + if table.path.stem == "Cells": + table.rows = tuple((*row[:2], "0") for row in table.rows) + elif table.path.stem == "Nuclei": + table.rows = tuple((*row[:2], "0") for row in table.rows) + else: + table.rows = tuple((*row[:6], "0", *row[7:]) for row in table.rows) + result = RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + assert result.correlated_relationships is not None + assert all( + not graph.source_keys for graph in result.correlated_relationships.values() + ) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.rows = tuple((*row[:7], "999", *row[8:]) for row in table.rows) + with pytest.raises(ValueError, match="absent child"): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +@pytest.mark.parametrize( + "feature, identity", + [ + ("Children_ Nuclei _Count", "Nuclei"), + ("Children__Count", None), + ("Children_Nuclei", None), + (" children_Nuclei_Count", None), + ], +) +def test_child_count_parser_and_core_declaration_share_original_grammar( + feature: str, identity: str | None +) -> None: + from openhcs.core.runtime_relationships import ChildCountFeatureDeclaration + from openhcs.interop.cellprofiler.measurement_lookup import ( + CellProfilerChildCountFeatureParser, + ) + + assert ChildCountFeatureDeclaration.from_feature_name(feature) == identity + parsed = CellProfilerChildCountFeatureParser().parse_feature(feature) + assert (None if parsed is None else parsed.object_name) == identity + + +@pytest.mark.parametrize("swapped", (False, True)) +def test_physical_inventory_consumes_redundant_edges_and_compares_exact_correlations( + tmp_path: Path, swapped: bool +) -> None: + from benchmark.matched_cellprofiler_batch import _require_compared_output_inventory + from openhcs.core.runtime_exports import RuntimeExportObservation + + reference = _saved_outputs(tmp_path / "native", edges=False) + candidate = _saved_outputs(tmp_path / "saved", edges=True, swapped=swapped) + reference_exports = RuntimeExportObservation.from_output_root(tmp_path / "native") + candidate_exports = RuntimeExportObservation.from_output_root(tmp_path / "saved") + kwargs = dict( + reference_files=frozenset((tmp_path / "native").iterdir()), + candidate_files=frozenset((tmp_path / "saved").iterdir()), + reference_exports=reference_exports, + candidate_exports=candidate_exports, + reference_snapshot=reference, + candidate_snapshot=candidate, + ) + if swapped: + with pytest.raises(RuntimeError, match="correlations differ"): + _require_compared_output_inventory(**kwargs) + else: + _require_compared_output_inventory(**kwargs) + + +@pytest.mark.parametrize( + "parent_value, message", + [("999", "absent parent endpoint"), ("2.5", "integral endpoint")], +) +def test_native_parent_rows_require_real_integral_parent_domain( + tmp_path: Path, parent_value: str, message: str +) -> None: + snapshot = _saved_outputs(tmp_path / "native", edges=False) + table = next(table for table in snapshot.tables if table.path.stem == "Nuclei") + table.rows = tuple((*row[:2], parent_value) for row in table.rows) + with pytest.raises(ValueError, match=message): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_native_child_count_permutation_cannot_hide_in_equal_histograms( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "native", edges=False) + table = next(table for table in snapshot.tables if table.path.stem == "Cells") + table.rows = tuple((*row[:2], "0" if row[1] == "1" else "1") for row in table.rows) + with pytest.raises(ValueError, match="disagree with.*children_nuclei_count"): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_snapshot_json_cache_retains_correlation_scope(tmp_path: Path) -> None: + import json + + known = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "saved", edges=True) + ) + for snapshot in (known, RuntimeMeasurementSnapshot(known.measurement_fact_counts)): + restored = RuntimeMeasurementSnapshot.from_cache_payload( + json.loads(json.dumps(snapshot.to_cache_payload())) + ) + assert restored.correlated_relationships == snapshot.correlated_relationships + assert restored.measurement_fact_counts == snapshot.measurement_fact_counts + + +@pytest.mark.parametrize("slice_index", (None, 0, 2)) +def test_object_identity_owns_required_plane_admission(slice_index: int | None) -> None: + from openhcs.core.runtime_relationships import ObjectInstanceKey + + key = ObjectInstanceKey(1, slice_index) + if slice_index is None: + with pytest.raises(ValueError, match="explicit image or slice identity"): + key.required_slice_index() + else: + assert key.required_slice_index() == slice_index + + +def test_full_saved_comparison_cannot_admit_unknown_relationship_scope() -> None: + unknown = RuntimeMeasurementSnapshot({}) + with pytest.raises(RuntimeError, match="requires known relationship correlations"): + unknown.required_relationship_correlations() + known = RuntimeMeasurementSnapshot({}, {}) + assert known.required_relationship_correlations() is known.correlated_relationships diff --git a/tests/unit/test_file_materialization_options.py b/tests/unit/test_file_materialization_options.py index d4ee275f12..2a97847618 100644 --- a/tests/unit/test_file_materialization_options.py +++ b/tests/unit/test_file_materialization_options.py @@ -85,7 +85,7 @@ def test_source_identity_image_file_projects_addressable_stack_planes() -> None: assert primary_path == "/analysis/site1_saved.npy" assert spec.emits_variable_component_planes(payload) assert tuple( - identity.path for identity in spec.emitted_source_identities(payload) + identity.path for identity in spec.stream_source_identities(payload) ) == tuple(f"/input/site{index}.tif" for index in range(1, 4)) for index, plane in enumerate(planes, start=1): np.testing.assert_array_equal( diff --git a/tests/unit/test_function_artifact_materialization.py b/tests/unit/test_function_artifact_materialization.py index bca6321c5a..2a889182ae 100644 --- a/tests/unit/test_function_artifact_materialization.py +++ b/tests/unit/test_function_artifact_materialization.py @@ -119,13 +119,10 @@ StreamingOnlyArtifactMaterializationTargetPlan, actual_materialization_records, materialize_artifact_outputs, - materialized_artifact_output_paths, - observed_materialized_artifact_locations_by_address, - observed_materialized_artifact_output_paths, + preview_reused_materialized_artifact_locations, planned_materialization_preview, runtime_artifact_materializations, runtime_artifact_materializations_from_records, - runtime_export_artifact_output_paths, ) from openhcs.core.steps.function_output_identity import ( IncompleteFunctionOutputFilenameIdentityError, @@ -155,6 +152,7 @@ from openhcs.processing.materialization.core import ( MaterializationSpec, Output, + SavedMaterializationOutputs, materialization_outputs, ) from openhcs.processing.materialization.options import ( @@ -789,10 +787,10 @@ def test_materialize_artifact_outputs_uses_runtime_store_payload( def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -839,10 +837,10 @@ def test_materialize_artifact_outputs_attaches_image_schema_provenance(monkeypat def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -922,10 +920,10 @@ def fake_materialize(spec, data, path, manager, *_args, **kwargs): variable_components=kwargs["variable_components"], ) ) - return output_paths[0] + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -940,7 +938,14 @@ def fake_materialize(spec, data, path, manager, *_args, **kwargs): expected_paths = tuple( Path(f"/images/A01_s{site:03d}_w1_z001_t001_saved.tif") for site in range(1, 4) ) - assert materialized_artifact_output_paths(plan, context) == expected_paths + assert ( + tuple( + Path(output.path) + for materialization in runtime_artifact_materializations(plan, context) + for output in materialization.outputs(plan, context) + ) + == expected_paths + ) materialize_artifact_outputs( filemanager, @@ -990,10 +995,10 @@ def test_materialize_artifact_outputs_does_not_require_vfs_payload_for_store_rec def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -1045,10 +1050,10 @@ def test_materialize_artifact_outputs_uses_runtime_record_identity_not_final_pat def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -1099,10 +1104,10 @@ def test_materialize_artifact_outputs_uses_declared_measurement_csv_spec( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -1711,17 +1716,19 @@ def record_for_site(site: str) -> None: current_execution_records = context.runtime_value_store.observed_values_after( current_execution_cursor ) - assert observed_materialized_artifact_output_paths( - plan, - context, - current_execution_records, + assert tuple( + Path(location.path) + for locations in preview_reused_materialized_artifact_locations( + plan, context, current_execution_records + ).values() + for location in locations ) == ( Path( "/analysis/A01_site-2_z_index-1_timepoint-1_" "cell_counts_step7_details.csv" ), ) - assert observed_materialized_artifact_locations_by_address( + assert preview_reused_materialized_artifact_locations( plan, context, current_execution_records, @@ -1794,7 +1801,7 @@ def test_terminal_persistence_is_reported_without_becoming_declared_export() -> persistent_backend="disk", ) - locations = observed_materialized_artifact_locations_by_address( + locations = preview_reused_materialized_artifact_locations( plan, context, (record,), @@ -1808,7 +1815,6 @@ def test_terminal_persistence_is_reported_without_becoming_declared_export() -> ), ), } - assert runtime_export_artifact_output_paths(plan, context) == () assert not output_plan.materialization.participates_in_runtime_export_observation() @@ -1929,10 +1935,10 @@ def test_materialize_artifact_outputs_unions_measurement_subject_records( def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((tuple(data.iter_row_mappings()), path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2232,10 +2238,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2298,10 +2304,10 @@ def test_materialize_artifact_outputs_uses_actual_group_records(monkeypatch): def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2578,10 +2584,10 @@ def test_materialize_artifact_outputs_uses_group_measurement_artifact_identity( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2642,10 +2648,10 @@ def test_materialize_artifact_outputs_keeps_grouped_artifact_record_path( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2708,10 +2714,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2778,10 +2784,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2928,10 +2934,10 @@ def test_materialize_artifact_outputs_uses_declared_metadata_json_spec( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2962,9 +2968,10 @@ def test_materialize_artifact_outputs_skips_special_without_explicit_spec( def fake_materialize(*args, **kwargs): materialized.append((args, kwargs)) + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3057,9 +3064,10 @@ def test_materialize_artifact_outputs_skips_explicitly_disabled_artifact_without def fake_materialize(*args, **kwargs): materialized.append((args, kwargs)) + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3097,10 +3105,10 @@ def test_materialize_artifact_outputs_uses_declared_object_labels_roi_spec(monke def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3154,10 +3162,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3243,9 +3251,10 @@ def fake_materialize( **_kwargs, ): materialized.append((backends, backend_kwargs)) + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3307,10 +3316,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3389,10 +3398,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3451,10 +3460,10 @@ def test_materialize_artifact_outputs_uses_runtime_plane_group_identity( def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3522,10 +3531,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3623,10 +3632,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3795,10 +3804,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3898,10 +3907,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3984,10 +3993,10 @@ def test_materialize_image_uses_declared_filename_source_identity(monkeypatch): def fake_materialize(_spec, _data, path, *_args, **_kwargs): materialized.append(path) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -4144,7 +4153,11 @@ def test_compiled_z_axis_reaches_source_named_image_materialization() -> None: assert image_payload_metadata(saved_payload).plane_axis is ( RuntimePlaneAxis.RUNTIME_SLICE ) - assert materialized_artifact_output_paths(plan, context) == ( + assert tuple( + Path(output.path) + for materialization in runtime_artifact_materializations(plan, context) + for output in materialization.outputs(plan, context) + ) == ( Path("/images/A01_s001_w2_z001_t001_NucleiLabels.tif"), Path("/images/A01_s001_w2_z002_t001_NucleiLabels.tif"), ) @@ -4214,10 +4227,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index 5cda2912d1..7bbeb49d34 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -1704,7 +1704,7 @@ def enhance(image, *, runtime): } -def test_pattern_group_runtime_stacks_nominal_scalar_rgb_output_as_one_slice(): +def test_pattern_group_runtime_retains_nominal_scalar_rgb_output_as_one_image(): scalar_output = ImagePayloadMetadata( source_path="/input/A01_s1_w3.tif", source_component_metadata={ @@ -1767,10 +1767,10 @@ def rgb_output(image): artifact_kind=ImageArtifactType.value, ), ) - assert image_payload_data(output.stack_payload).shape == (1, 2, 3, 3) + assert image_payload_data(output.stack_payload).shape == (2, 3, 3) stack_metadata = image_payload_metadata(output.stack_payload) - assert stack_metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE - assert stack_metadata.source_channel_axis == 3 + assert stack_metadata.plane_axis is None + assert stack_metadata.source_channel_axis == -1 def test_pattern_group_runtime_uses_declared_output_slice_cardinality(): diff --git a/tests/unit/test_function_outputs.py b/tests/unit/test_function_outputs.py index 3e2a76a8fc..98d2cad9f9 100644 --- a/tests/unit/test_function_outputs.py +++ b/tests/unit/test_function_outputs.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from dataclasses import replace from types import SimpleNamespace import numpy as np @@ -39,6 +40,7 @@ from openhcs.core.runtime_image_values import ( ImageMetadataPayload, ImagePayloadMetadata, + image_payload_metadata, ) from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.source_image_provenance import ( @@ -52,6 +54,7 @@ SOURCE_VOXEL_SPACING_FIELD, SOURCE_VOXEL_SPACING_UNIT_FIELD, SourceVoxelSpacing, + SourceVoxelSpacingUnit, ) from openhcs.core.source_projection import SourceProjectionMetadataSerializer from openhcs.core.source_spatial_domain import SourceSpatialDomain @@ -66,6 +69,10 @@ ProducedOutputSemantics, step_output_manifest, ) +from openhcs.core.steps.function_artifact_materialization import ( + MaterializedRuntimeArtifact, + RuntimeArtifactMaterialization, +) from openhcs.core.steps.function_outputs import ( MaterializedImageOutputWriter, MemoryOutputWriter, @@ -97,6 +104,7 @@ MaterializedFilenameIdentity, ) + @pytest.mark.parametrize("backend", [Backend.ZARR.value, "custom-array-store"]) def test_memory_output_writer_projects_runtime_image_payload_for_array_storage( backend, @@ -137,7 +145,9 @@ def test_function_step_metadata_follows_runtime_artifact_persistence( monkeypatch.setattr( authority, method_name, - lambda _context, _plan, event=event: events.append(event), + lambda _context, _plan, event=event, **_kwargs: (events.append(event), ())[ + 1 + ], ) finalize_function_step_outputs(context, plan) @@ -1371,10 +1381,13 @@ def test_image_persistence_skips_object_label_main_flow_payloads(): assert ProducedMemoryPathsAuthority.paths(context, plan) == [] -def test_metadata_target_family_discovers_new_declaration_without_consumer_edits(tmp_path): +def test_metadata_target_family_discovers_new_declaration_without_consumer_edits( + tmp_path, +): registry = OpenHCSMetadataWriter.OutputTarget.__registry__ original_keys = set(registry) try: + class SupplementalImageMetadataTarget(OpenHCSMetadataWriter.OutputTarget): @classmethod def from_plan(cls, plan): @@ -1391,7 +1404,9 @@ def from_plan(cls, plan): context = context_stub(FileManagerStub({})) (target,) = OpenHCSMetadataWriter.OutputTarget.for_plan(plan) assert type(target) is SupplementalImageMetadataTarget - assert OpenHCSMetadataWriter.OutputTarget.for_execution(context, plan) == (target,) + assert OpenHCSMetadataWriter.OutputTarget.for_execution(context, plan) == ( + target, + ) assert target.produced_projection_metadata(context, plan) is None finally: for key in set(registry) - original_keys: @@ -1638,13 +1653,16 @@ def test_runtime_image_artifact_projects_persisted_source_binding( ), outputs=lambda _plan, _context, **_kwargs: (output,), ) - monkeypatch.setattr( - "openhcs.core.steps.function_outputs.runtime_artifact_materializations", - lambda _plan, _context: (materialization,), + saved_artifacts = ( + MaterializedRuntimeArtifact( + outputs_by_backend={Backend.DISK.value: (output,)}, + materialization=materialization, + ), ) target = MaterializedImageMetadataTarget.from_plan(plan) assert target is not None + target = replace(target, artifact_materializations=saved_artifacts) [(projection, virtual_path)] = target.runtime_artifact_projection_paths( context, plan ) @@ -1755,9 +1773,11 @@ def test_runtime_multiplane_label_artifact_projects_persisted_source_binding( ), outputs=lambda _plan, _context, **_kwargs: (output,), ) - monkeypatch.setattr( - "openhcs.core.steps.function_outputs.runtime_artifact_materializations", - lambda _plan, _context: (materialization,), + saved_artifacts = ( + MaterializedRuntimeArtifact( + outputs_by_backend={Backend.DISK.value: (output,)}, + materialization=materialization, + ), ) output_plan = ArtifactOutputPlan( @@ -1769,6 +1789,7 @@ def test_runtime_multiplane_label_artifact_projects_persisted_source_binding( plan.artifact_outputs = {output_plan.ref(): output_plan} target = RuntimeArtifactMetadataTarget.from_plan(plan) assert target is not None + target = replace(target, artifact_materializations=saved_artifacts) [(projection, virtual_path)] = target.runtime_artifact_projection_paths( context, plan ) @@ -1933,10 +1954,12 @@ def test_produced_address_publication_never_parses_generated_filenames( output_dir = plate_root / "images" output_dir.mkdir(parents=True) parser = SourceSchemaFilenameParser() - filemanager = FileManager({ - Backend.DISK.value: DiskStorageBackend(), - Backend.MEMORY.value: MemoryStorageBackend(), - }) + filemanager = FileManager( + { + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + } + ) context = context_stub(filemanager, parser=parser) context.metadata_cache = { AllComponents.WELL: {well: None, "not-produced": None}, @@ -1962,12 +1985,17 @@ def test_produced_address_publication_never_parses_generated_filenames( metadata = ImagePayloadMetadata( source_path=source_path, source_component_metadata={ - "well": well, "site": 1, "channel": 1, - "z_index": z_index, "timepoint": 1, + "well": well, + "site": 1, + "channel": 1, + "z_index": z_index, + "timepoint": 1, }, source_voxel_spacing=SourceVoxelSpacing((0.5, 0.5)), ) - identity = FunctionOutputIdentityAuthority.identity_from_metadata(parser, metadata) + identity = FunctionOutputIdentityAuthority.identity_from_metadata( + parser, metadata + ) assert identity is not None assert identity.extension == extension identity = identity.with_filename_qualifier("centre_dots") @@ -1980,14 +2008,20 @@ def test_produced_address_publication_never_parses_generated_filenames( assert parsed.value_for(AllComponents.WELL) == well path = output_dir / filename tifffile.imwrite(path, pixels) - filemanager.save(metadata.payload_with(pixels, None), str(path), Backend.MEMORY.value) - records.append(ProducedOutputSemantics.from_output( - plan, path, identity, - output_context=AlignedImageSliceContext.main_flow( - output_key="centre_dots", artifact_kind=ImageArtifactType.value - ), - image_metadata=metadata, - )) + filemanager.save( + metadata.payload_with(pixels, None), str(path), Backend.MEMORY.value + ) + records.append( + ProducedOutputSemantics.from_output( + plan, + path, + identity, + output_context=AlignedImageSliceContext.main_flow( + output_key="centre_dots", artifact_kind=ImageArtifactType.value + ), + image_metadata=metadata, + ) + ) paths.append(f"images/{filename}") step_output_manifest(context).record_outputs(plan, records) @@ -2029,11 +2063,15 @@ def reject_generated_path_parse(_filename): removed = paths[0] (plate_root / removed).unlink() OpenHCSMetadataWriter.finalize_completed_plate({well: context}) - reconciled = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + reconciled = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES][ + "images" + ] assert set(reconciled[FIELDS.IMAGE_FILES]) == set(paths[1:]) assert removed not in reconciled[FIELDS.WORKSPACE_MAPPING] assert removed not in reconciled[FIELDS.SOURCE_METADATA] - assert {record["virtual_path"] for record in reconciled[FIELDS.SOURCE_PROJECTION]} == set(paths[1:]) + assert { + record["virtual_path"] for record in reconciled[FIELDS.SOURCE_PROJECTION] + } == set(paths[1:]) assert set(reconciled["z_indexes"]) == {str(z) for z in z_values[1:]} @@ -2044,8 +2082,7 @@ def test_produced_stacked_dotted_identity_keeps_declared_extension( ): parser = SourceSchemaFilenameParser() source_paths = tuple( - f"/source/{well}_s001_w1_z{z:03d}_t001{extension}" - for z in (3, 1, 2) + f"/source/{well}_s001_w1_z{z:03d}_t001{extension}" for z in (3, 1, 2) ) metadata = ImagePayloadMetadata( source_image_provenance_planes=SourceImageProvenancePlanes.from_components( @@ -2055,7 +2092,9 @@ def test_produced_stacked_dotted_identity_keeps_declared_extension( request = FunctionOutputPathRequest( parser=parser, output_dir=tmp_path, - output_payload=metadata.payload_with(np.zeros((3, 4, 5), dtype=np.uint16), None), + output_payload=metadata.payload_with( + np.zeros((3, 4, 5), dtype=np.uint16), None + ), input_path=Path(source_paths[0]).name, variable_components=(VariableComponents.Z_INDEX,), ) @@ -2176,7 +2215,9 @@ def test_runtime_image_metadata_target_requires_persisted_images(tmp_path, conte if contents == "tables": (directory / "measurements.csv").write_text("ObjectNumber,Area\n1,4\n") else: - tifffile.imwrite(directory / "A01_s001_w2_z001_t001.tif", np.ones((2, 2), dtype=np.uint8)) + tifffile.imwrite( + directory / "A01_s001_w2_z001_t001.tif", np.ones((2, 2), dtype=np.uint8) + ) context = context_stub(FileManager({Backend.DISK.value: DiskStorageBackend()})) plan = function_step_plan("SaveImages") plan.write_backend = Backend.MEMORY.value @@ -2223,7 +2264,32 @@ def test_runtime_image_metadata_target_requires_persisted_images(tmp_path, conte target = RuntimeArtifactMetadataTarget.from_plan(plan) assert target is not None assert target.output_dir == directory - selected = OpenHCSMetadataWriter.OutputTarget.for_execution(context, plan) + saved_artifacts = () + if contents == "images": + record = context.runtime_value_store.values()[0] + materialization = RuntimeArtifactMaterialization.from_record( + output_plan=output_plan, + record=record, + plan=plan, + context=context, + ) + saved_artifacts = ( + MaterializedRuntimeArtifact( + outputs_by_backend={ + Backend.DISK.value: ( + Output.from_metadata( + path=str(directory / "A01_s001_w2_z001_t001.tif"), + content=record.value.data, + metadata=image_payload_metadata(record.value.data), + ), + ) + }, + materialization=materialization, + ), + ) + selected = OpenHCSMetadataWriter.OutputTarget.for_execution( + context, plan, artifact_materializations=saved_artifacts + ) if contents == "images": assert selected == (target,) else: @@ -2295,8 +2361,12 @@ def test_declared_image_destinations_publish_and_reconcile_after_value_cleanup( path=output_plan.path, backend=Backend.MEMORY.value, ) - RuntimeArtifactMaterializationAuthority.materialize(context, plan) - OpenHCSMetadataWriter.write(context, plan) + materializations = RuntimeArtifactMaterializationAuthority.materialize( + context, plan + ) + OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ) context.runtime_value_store.clear() if stray_image: tifffile.imwrite(tmp_path / "images/review/unowned.tif", pixels) @@ -2319,3 +2389,135 @@ def test_declared_image_destinations_publish_and_reconcile_after_value_cleanup( assert projection.component_value(AllComponents.CHANNEL) == "2" assert projection.source_metadata["site"] == "1" np.testing.assert_array_equal(tifffile.imread(tmp_path / path), pixels) + + +@pytest.mark.parametrize("with_raster", [False, True]) +def test_actual_array_exports_only_publish_declared_raster_inventory( + tmp_path, with_raster +): + """A saved NumPy array is an actual output, without being a raster inventory.""" + filemanager = FileManager( + { + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + } + ) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + context.runtime_value_store = RuntimeValueStore() + context.metadata_cache = {} + context.tiff_config = None + plan = function_step_plan("Independent array and raster exports") + plan.streaming_configs = {} + plan.write_backend = Backend.MEMORY.value + plan.output_plate_root = str(tmp_path) + plan.output_dir = tmp_path / "images" + plan.sub_dir = "images" + plan.analysis_results_dir = str(tmp_path / "results") + plan.create_openhcs_metadata = True + plan.runtime_artifact_materialization = RuntimeArtifactMaterializationPlan( + persistent_enabled=True, + persistent_backend=Backend.DISK.value, + ) + context.step_plans = {plan.step_index: plan} + pixels = np.arange(20, dtype=np.float32).reshape(4, 5) / 7 + payload = ImageMetadataPayload( + pixels, + ImagePayloadMetadata( + source_component_metadata={ + "well": "A01", + "site": "1", + "channel": "2", + "z_index": "1", + "timepoint": "1", + }, + source_dtype="float32", + source_voxel_spacing=SourceVoxelSpacing( + (0.65, 0.65), SourceVoxelSpacingUnit.MICROMETERS + ), + ), + ) + declarations = [("NumericField", "exports/Illum.npy")] + if with_raster: + declarations.append(("RasterField", "exports/A01_s001_w2_z001_t001.tif")) + for name, relative_path in declarations: + output_plan = ArtifactOutputPlan( + name=name, + path=f"/memory/{name}.pkl", + artifact_type=ImageArtifactType, + materialization=MaterializationSpec( + ImageFileOptions( + filename_suffix=Path(relative_path).suffix, + relative_path_template=relative_path, + ) + ), + ) + plan.artifact_outputs[output_plan.ref()] = output_plan + context.runtime_value_store.record( + RuntimeValue.normalize(output_plan, payload, axis_id="A01"), + path=output_plan.path, + backend=Backend.MEMORY.value, + ) + + materializations = RuntimeArtifactMaterializationAuthority.materialize( + context, plan + ) + saved_outputs = tuple( + output + for artifact in materializations + for output in artifact.outputs_for_backend(Backend.DISK.value) + ) + expected_paths = { + tmp_path / "images" / relative_path for _name, relative_path in declarations + } + assert {Path(output.path) for output in saved_outputs} == expected_paths + assert all(path.is_file() for path in expected_paths) + np.testing.assert_array_equal( + np.load(tmp_path / "images/exports/Illum.npy", allow_pickle=False), pixels + ) + observed_locations = tuple( + location + for artifact in materializations + for locations in ( + artifact.observation(plan).materialized_locations_by_address.values() + ) + for location in locations + ) + assert {Path(location.path) for location in observed_locations} == expected_paths + + OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ) + context.runtime_value_store.clear() + OpenHCSMetadataWriter.finalize_completed_plate({"A01": context}) + metadata_path = tmp_path / "openhcs_metadata.json" + subdirectories = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES] + assert set(subdirectories) == {"images/exports"} + subdirectory = subdirectories["images/exports"] + assert subdirectory[SourceProjectionMetadataSerializer.RESULTS_DIR_FIELD] == ( + "images/exports" + ) + if not with_raster: + assert subdirectory[FIELDS.IMAGE_FILES] == [] + assert ( + VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries + == {} + ) + return + np.testing.assert_array_equal( + tifffile.imread(tmp_path / "images/exports/A01_s001_w2_z001_t001.tif"), pixels + ) + assert subdirectory[FIELDS.IMAGE_FILES] == [ + "images/exports/A01_s001_w2_z001_t001.tif" + ] + entries = VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries + raster = entries["images/exports/A01_s001_w2_z001_t001.tif"] + assert raster.source_alias == "RasterField" + assert raster.image_metadata.source_voxel_spacing.values_zyx == (0.65, 0.65) + assert ( + raster.image_metadata.source_voxel_spacing.native_coordinate_unit + == "micrometer" + ) diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index d031997b88..fc4fc52482 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -3733,6 +3733,9 @@ def test_producer_anchored_pipeline_start_paths_use_exact_source_projection_bund """Producer bookkeeping must not override exact workspace source ownership.""" from openhcs.core.steps import function_runtime + from openhcs.core.runtime_source_binding_cache import ( + RuntimeSourceBindingContextCache, + ) virtual_paths = ( "A01_s001_w1_z001_t001.tif", @@ -3870,6 +3873,7 @@ def resolve_address(backend_address, backend, *, base_path): ), filemanager=SourceFileManager(), runtime_image_stack_cache=RuntimeImageStackCache(), + runtime_source_binding_context_cache=RuntimeSourceBindingContextCache(), runtime_source_workspace_projection_cache=( VirtualWorkspaceSourceProjectionCache() ), @@ -5340,8 +5344,11 @@ def save_batch( def qualified_producer_manifest(tmp_path): parser = SourceSchemaFilenameParser() producer = SimpleNamespace( - step_scope_id="producer", step_name="Producer", pipeline_position=0, - axis_id="A01", output_dir=tmp_path, + step_scope_id="producer", + step_name="Producer", + pipeline_position=0, + axis_id="A01", + output_dir=tmp_path, ) consumer = SimpleNamespace( axis_id="A01", @@ -5355,14 +5362,22 @@ def qualified_producer_manifest(tmp_path): parsed = parser.parse_filename(f"A01_s001_w2_z{plane:03d}_t001.tif") identity = FunctionOutputIdentity( component_values=dict(parsed.component_wire_mapping()), - extension=".tif", source="test", filename_qualifier=f"Output{plane}", + extension=".tif", + source="test", + filename_qualifier=f"Output{plane}", + ) + records.append( + ProducedOutputSemantics.from_output( + producer, + tmp_path + / FunctionOutputPathAuthority.filename_for_identity(parser, identity), + identity, + output_context=AlignedImageSliceContext.main_flow( + output_key=f"Output{plane}", + artifact_kind=ImageArtifactType.value, + ), + ) ) - records.append(ProducedOutputSemantics.from_output( - producer, tmp_path / FunctionOutputPathAuthority.filename_for_identity(parser, identity), - identity, output_context=AlignedImageSliceContext.main_flow( - output_key=f"Output{plane}", artifact_kind=ImageArtifactType.value, - ), - )) store = StepOutputManifestStore() store.begin_step(producer) store.record_outputs(producer, records) @@ -5370,7 +5385,8 @@ def qualified_producer_manifest(tmp_path): def test_step_output_manifest_batch_lookup_preserves_aliases_order_and_duplicates( - qualified_producer_manifest, monkeypatch, + qualified_producer_manifest, + monkeypatch, ): store, _producer, consumer, records, parser = qualified_producer_manifest calls = [] @@ -5380,7 +5396,9 @@ def count_filename(parser, identity): calls.append(identity) return original(parser, identity) - monkeypatch.setattr(FunctionOutputPathAuthority, "filename_for_identity", count_filename) + monkeypatch.setattr( + FunctionOutputPathAuthority, "filename_for_identity", count_filename + ) paths = ( records[1].output_path, "A01_s001_w2_z001_t001.tif", @@ -5389,7 +5407,10 @@ def count_filename(parser, identity): ) result = store.producer_output_contexts_for_paths(consumer, paths, parser) assert tuple(context.output_key for context in result) == ( - "Output2", "Output1", "Output2", "Output1", + "Output2", + "Output1", + "Output2", + "Output1", ) assert len(calls) == len(records) @@ -5407,12 +5428,17 @@ def test_step_output_manifest_batch_lookup_template_deduplicates_record_aliases( assert index.matching_records(path) == (records[0],) -@pytest.mark.parametrize("path, count", [ - ("missing.tif", 0), - ("A01_s001_w2_z{plane}_t001.tif", 2), -]) +@pytest.mark.parametrize( + "path, count", + [ + ("missing.tif", 0), + ("A01_s001_w2_z{plane}_t001.tif", 2), + ], +) def test_step_output_manifest_batch_lookup_rejects_missing_and_ambiguous_templates( - qualified_producer_manifest, path, count, + qualified_producer_manifest, + path, + count, ): store, _producer, consumer, _records, parser = qualified_producer_manifest with pytest.raises(NoStepOutputManifestMatch, match=f"found {count}"): @@ -5425,10 +5451,12 @@ def test_step_output_manifest_batch_lookup_rejects_shared_basename( store, producer, consumer, records, parser = qualified_producer_manifest alias = records[0].relative_output_path other = ProducedOutputSemantics.from_output( - producer, producer.output_dir / "another" / alias, + producer, + producer.output_dir / "another" / alias, FunctionOutputIdentity( component_values=records[1].component_values, - extension=".tif", source="test", + extension=".tif", + source="test", ), output_context=records[1].output_context, ) diff --git a/tests/unit/test_graph_roi_source_roundtrip_134.py b/tests/unit/test_graph_roi_source_roundtrip_134.py new file mode 100644 index 0000000000..83f0b9b478 --- /dev/null +++ b/tests/unit/test_graph_roi_source_roundtrip_134.py @@ -0,0 +1,272 @@ +"""Receiving #134: actual graph context/writer/ZIP, without a native process.""" + +from pathlib import Path + +import numpy as np +import pytest +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import load_rois_from_zip + +from openhcs.core.artifacts import ( + ArtifactOutputPlan, + ArtifactSpec, + ObjectArtifactMemberSubjectRelation, + ObjectArtifactSubjectBinding, + ObjectLabelsArtifactType, + SpatialGraphArtifactType, +) +from openhcs.core.config import NapariStreamingConfig +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.runtime_spatial_graph import ( + SpatialGraph, + SpatialGraphEdge, + SpatialGraphNode, +) +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.steps.function_runtime import FunctionOutputContextStrategy +from openhcs.core.viewer_streaming_service import RoiStreamingRequest, StreamingService +from openhcs.processing.materialization import ( + MaterializationSpec, + SpatialGraphROIOptions, + materialization_outputs, + materialize, +) + + +@pytest.fixture(autouse=True) +def no_optional_storage_bootstrap(monkeypatch): + import polystore.base + + monkeypatch.setattr( + polystore.base, + "ensure_storage_registry", + lambda: pytest.fail("Provider/bootstrap execution is forbidden in this test"), + ) + + +class ProjectionAudit: + """Independent capability exercises cooperative construction and projection.""" + + def __init__(self, **kwargs): + self.projection_events = [] + super().__init__(**kwargs) + + def contextualized_source_provenance(self, provenance): + self.projection_events.append("before") + result = super().contextualized_source_provenance(provenance) + self.projection_events.append("after") + return result + + +class AuditedGraph(ProjectionAudit, SpatialGraph): + """New declaration; no writer, context strategy or reader registration.""" + + +def graph_and_plan(*, plane, graph_type=SpatialGraph, extra_features=None): + nodes = tuple( + SpatialGraphNode(index + 1, coordinate) + for index, coordinate in enumerate(((2.25, 3.5), (5.25, 9.5), (7.25, 12.5))) + ) + edges = tuple( + SpatialGraphEdge.from_features( + edge_id=index + 1, + source=nodes[index], + target=nodes[index + 1], + coordinates=np.asarray( + (nodes[index].coordinates, nodes[index + 1].coordinates) + ), + features={ + "neuron_label": 7, + "branch_distance_um": 8.5, + **(extra_features or {}), + }, + ) + for index in range(2) + ) + graph = graph_type( + name="declared_graph", + nodes=nodes, + edges=edges, + coordinate_spacing=(1.3556, 1.3556), + source_plane_index=plane, + ) + neurons = ArtifactSpec.output("neurons", ObjectLabelsArtifactType) + plan = ArtifactOutputPlan( + name=graph.name, + path="/engineering/graph.pkl", + artifact_type=SpatialGraphArtifactType, + relations=( + ObjectArtifactMemberSubjectRelation( + source=neurons.ref(), member_id_field="neuron_label" + ), + ), + producer_step_index=4, + producer_step_scope_id="engineering-step", + ) + return graph, plan + + +def contextualize(graph, plan): + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/engineering/nuclear.tif", "/engineering/body.tif"), + component_metadata=tuple( + { + "well": "A01", + "site": 1, + "channel": channel, + "z_index": 1, + "timepoint": 1, + } + for channel in (1, 2) + ), + ), + source_image_names=("nuclear", "body"), + source_voxel_spacing=SourceVoxelSpacing(graph.coordinate_spacing), + ) + source = metadata.payload_with(np.zeros((2, 16, 16), dtype=np.uint8)) + result = FunctionOutputContextStrategy.for_output_plan(plan).contextualize( + source, graph, plan, None + ) + assert result.source_provenance == metadata.source_provenance.for_source_plane( + graph.source_plane_index + ) + return result + + +def save_graph(tmp_path, graph, plan): + manager = FileManager({"disk": DiskStorageBackend()}) + path = materialize( + MaterializationSpec(SpatialGraphROIOptions()), + data=graph, + path=str(tmp_path / "misleading_w99_z999.roi.zip"), + filemanager=manager, + backends=["disk"], + backend_kwargs={}, + output_plan=plan, + ) + return Path(path), load_rois_from_zip(Path(path)) + + +@pytest.mark.parametrize("plane", [0, 1]) +def test_declared_source_plane_survives_graph_writer_zip_and_subject_features( + tmp_path, plane +): + graph, plan = graph_and_plan(plane=plane) + graph = contextualize(graph, plan) + (output,) = materialization_outputs( + MaterializationSpec(SpatialGraphROIOptions()), + data=graph, + path=str(tmp_path / "graph.roi.zip"), + filemanager=FileManager({"disk": DiskStorageBackend()}), + output_plan=plan, + ) + path, restored = save_graph(tmp_path, graph, plan) + metadata = ROIArchiveSourceMetadata.decode(restored) + assert metadata is not None, "Native graph ZIP lost its existing source declaration" + assert output.metadata.source_provenance == graph.source_provenance + assert metadata.source_provenance == graph.source_provenance + assert ( + metadata.source_path + == ("/engineering/nuclear.tif", "/engineering/body.tif")[plane] + ) + assert metadata.source_component_metadata["channel"] == plane + 1 + assert metadata.source_voxel_spacing == SourceVoxelSpacing(graph.coordinate_spacing) + assert metadata.plane_axis is None + assert metadata.source_spatial_domain == output.metadata.source_spatial_domain + assert path.name == "misleading_w99_z999.graph.roi.zip" + geometry = ROIArchiveSourceMetadata.geometry(restored) + assert len(geometry) == len(graph.edges) + for roi, edge in zip(geometry, graph.edges, strict=True): + np.testing.assert_array_equal(roi.shapes[0].coordinates, edge.coordinates) + assert roi.metadata["edge_id"] == edge.edge_id + assert roi.metadata["source_node_id"] == edge.source_node_id + assert roi.metadata["target_node_id"] == edge.target_node_id + assert roi.metadata["neuron_label"] == 7 + assert roi.metadata["branch_distance_um"] == 8.5 + assert roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_ID_FEATURE] == 7 + assert ( + '"object_labels","neurons","engineering-step",4' + in roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_FEATURE] + ) + assert ROIArchiveSourceMetadata.FIELD not in roi.metadata + assert ( + len( + { + roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_FEATURE] + for roi in geometry + } + ) + == 1 + ) + + +def test_independent_graph_capability_and_new_feature_need_no_consumers(tmp_path): + original, plan = graph_and_plan( + plane=1, + graph_type=AuditedGraph, + extra_features={"engineering_confidence": 0.375}, + ) + graph = contextualize(original, plan) + assert original.projection_events == ["before", "after"] + assert ( + graph.projection_events == [] + ) # cooperative constructor for contextualized copy + _, restored = save_graph(tmp_path, graph, plan) + metadata = ROIArchiveSourceMetadata.decode(restored) + assert metadata is not None, "New graph declaration lost its source declaration" + assert metadata.source_provenance == graph.source_provenance + assert all(roi.metadata["engineering_confidence"] == 0.375 for roi in restored) + assert all( + ROIArchiveSourceMetadata.FIELD not in roi.metadata + for roi in ROIArchiveSourceMetadata.geometry(restored) + ) + + +@pytest.mark.parametrize("invalid", ["absent", "unbound", "mixed", "conflicting"]) +def test_original_native_admission_refuses_bad_graph_sources_before_transport( + tmp_path, invalid +): + graph, plan = graph_and_plan(plane=1) + graph = contextualize(graph, plan) + path, rois = save_graph(tmp_path, graph, plan) + geometry = ROIArchiveSourceMetadata.geometry(rois) + metadata = ImagePayloadMetadata(source_provenance=graph.source_provenance) + bad_archives = { + "absent": geometry, + "unbound": ROIArchiveSourceMetadata.bind(geometry, ImagePayloadMetadata()), + "mixed": [ + ROIArchiveSourceMetadata.bind(geometry[:1], metadata)[0], + geometry[1], + ], + "conflicting": [ + ROIArchiveSourceMetadata.bind(geometry[:1], metadata)[0], + ROIArchiveSourceMetadata.bind( + geometry[1:], + metadata.replace_fields(source_path="/engineering/other.tif"), + )[0], + ], + } + DiskStorageBackend().save(bad_archives[invalid], path) + # No viewer/handler object: original source admission must reject before using either. + service = StreamingService( + FileManager({"disk": DiskStorageBackend()}), None, tmp_path + ) + with pytest.raises( + ValueError, match="Native ROI source metadata|missing or conflicting" + ): + service.stream_rois( + RoiStreamingRequest( + viewer=None, + config=NapariStreamingConfig(enabled=True), + status_callback=lambda _: None, + error_callback=lambda error: pytest.fail(error), + roi_filenames=(path.name,), + require_source_metadata=True, + ) + ) diff --git a/tests/unit/test_image_payload_projection_algebra.py b/tests/unit/test_image_payload_projection_algebra.py new file mode 100644 index 0000000000..677ed7beb1 --- /dev/null +++ b/tests/unit/test_image_payload_projection_algebra.py @@ -0,0 +1,495 @@ +"""Physical source-plane projection preserves composition and mutable isolation.""" + +from dataclasses import fields + +import numpy as np +import pytest + +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + ImagePayloadSliceProjector, + ImageUnitIntervalIntensityMetadata, + image_payload_mask, + image_payload_metadata, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_image_provenance import ( + RuntimeSourceImageProvenancePlane, + SourceImageIdentity, + SourceImageProvenanceContributor, + SourceImageProvenancePlanes, +) +from openhcs.core.source_spatial_domain import SourceSpatialDomain + + +def _metadata(axis, channel_axis): + return ImagePayloadMetadata( + source_path="/tmp/parent.tif", + source_component_metadata={"well": "A01", "extension": ".tif"}, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/tmp/first.tif", "/tmp/second.tif"), + component_metadata=({"site": "1"}, {"site": "2"}), + ), + intensity_scale=255.0, + source_dtype="uint8", + source_plane_intensity_scales=(255.0, 65535.0), + source_plane_dtypes=("uint8", "uint16"), + unit_interval_intensity=ImageUnitIntervalIntensityMetadata( + scale=255, source_plane_scales=(255, 65535) + ), + source_image_names=("DNA", "RNA"), + source_channel_axis=channel_axis, + plane_axis=axis, + ) + + +@pytest.mark.parametrize("axis", tuple(RuntimePlaneAxis)) +@pytest.mark.parametrize("channel_axis", (None, 1, 3, -1)) +@pytest.mark.parametrize("plane", (0, 1)) +def test_physical_projection_agrees_with_separate_semantic_operations( + axis, channel_axis, plane +): + metadata = _metadata(axis, channel_axis) + sequential = metadata.for_source_plane(plane).without_leading_plane_axis() + projected = metadata.for_leading_source_plane(plane) + for member in fields(ImagePayloadMetadata): + assert getattr(projected, member.name) == getattr(sequential, member.name) + assert projected.source_path == ("/tmp/first.tif", "/tmp/second.tif")[plane] + assert projected.source_component_metadata["well"] == "A01" + assert projected.source_component_metadata["site"] == str(plane + 1) + assert projected.unit_interval_intensity.scale == (255, 65535)[plane] + assert projected.source_plane_intensity_scales == () + assert projected.source_plane_dtypes == () + assert projected.plane_axis is None + projected.source_provenance.source_identity.path = "/tmp/changed.tif" + assert metadata.source_path == "/tmp/parent.tif" + assert metadata.source_image_provenance_planes.paths[plane] != "/tmp/changed.tif" + + +def test_physical_projection_constructs_only_the_final_metadata(monkeypatch): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + calls = [] + original = ImagePayloadMetadata.__post_init__ + + def observe(self, *values): + calls.append(self) + original(self, *values) + + monkeypatch.setattr(ImagePayloadMetadata, "__post_init__", observe) + projected = metadata.for_leading_source_plane(1) + assert calls == [projected] + + +def test_physical_projection_rejects_leading_channel_axis(): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, 0) + with pytest.raises(ValueError, match="both plane and channel"): + metadata.for_leading_source_plane(1) + + +def test_physical_projection_requires_declared_plane_axis(): + metadata = _metadata(None, None) + with pytest.raises(ValueError, match="declared plane axis"): + metadata.for_leading_source_plane(1) + + +@pytest.mark.parametrize("channel_axis", (False, True, "1", 1.5)) +def test_physical_projection_preserves_invalid_mutated_channel_error(channel_axis): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + metadata.source_channel_axis = channel_axis + with pytest.raises(TypeError, match="must be int or None"): + metadata.for_source_plane(1).without_leading_plane_axis() + with pytest.raises(TypeError, match="must be int or None"): + metadata.for_leading_source_plane(1) + + +def test_physical_projection_preserves_invalid_mutated_plane_error(): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + metadata.plane_axis = "invalid-axis" + with pytest.raises(ValueError): + metadata.for_source_plane(1).without_leading_plane_axis() + with pytest.raises(ValueError): + metadata.for_leading_source_plane(1) + + +def test_projection_preserves_selected_spacing_and_nested_contributors(): + contributor = SourceImageProvenanceContributor( + SourceImageIdentity("/tmp/contributor.tif", {"site": "3"}), + source_image_name="Original", + ) + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_image_provenance_planes=SourceImageProvenancePlanes( + ( + RuntimeSourceImageProvenancePlane( + SourceImageIdentity( + "/tmp/selected.tif", + {"OpenHCSSourceVoxelSpacingZYX": "4,2,1"}, + ), + contributors=(contributor,), + source_image_name="DNA", + ), + ) + ), + ) + sequential = metadata.for_source_plane(0).without_leading_plane_axis() + projected = metadata.for_leading_source_plane(0) + for member in fields(ImagePayloadMetadata): + assert getattr(projected, member.name) == getattr(sequential, member.name) + assert projected.source_image_provenance_planes.planes == (contributor,) + + +def test_projection_observes_mask_and_metadata_mutation(): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + mask = np.ones((2, 3, 4), dtype=bool) + projector = ImagePayloadSliceProjector(mask, metadata) + data = np.zeros((3, 4)) + first = projector.payload_for_slice(data, 1) + assert image_payload_metadata(first).intensity_scale == 65535.0 + metadata.source_plane_intensity_scales = (1.0, 2.0) + mask[1, 0, 0] = False + second = projector.payload_for_slice(data, 1) + assert image_payload_metadata(second).intensity_scale == 2.0 + assert not image_payload_mask(second)[0, 0] + mask.resize((2, 2, 6), refcheck=False) + with pytest.raises(ValueError, match="cannot be projected"): + projector.payload_for_slice(data, 1) + + +def _metadata_with_observed_plane_fields(*, proof_raises=False): + events = [] + + class ObservedInt(int): + def __int__(self): + events.append("proof-int") + if proof_raises: + raise RuntimeError("proof conversion observed") + return 1 + + class ObservedTuple(tuple): + def __getitem__(self, index): + events.append("intensity-getitem") + return super().__getitem__(index) + + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_plane_intensity_scales=ObservedTuple((2.0,)), + unit_interval_intensity=ImageUnitIntervalIntensityMetadata( + source_plane_scales=(ObservedInt(1),) + ), + ) + return metadata, events + + +def test_plane_proof_failure_precedes_leading_channel_guard(): + metadata, events = _metadata_with_observed_plane_fields(proof_raises=True) + metadata.source_channel_axis = 0 + with pytest.raises(RuntimeError, match="proof conversion observed"): + metadata.for_leading_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + + +def test_plane_field_effects_precede_leading_channel_guard(): + metadata, events = _metadata_with_observed_plane_fields() + metadata.source_channel_axis = 0 + with pytest.raises(ValueError, match="both plane and channel"): + metadata.for_leading_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + + +def test_source_plane_preserves_intensity_then_proof_effect_order(): + metadata, events = _metadata_with_observed_plane_fields() + projected = metadata.for_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + assert projected.intensity_scale == 2.0 + assert projected.unit_interval_intensity.scale == 1 + + +def test_plane_proof_failure_precedes_mutated_channel_type_guard(): + metadata, events = _metadata_with_observed_plane_fields(proof_raises=True) + metadata.source_channel_axis = False + with pytest.raises(RuntimeError, match="proof conversion observed"): + metadata.for_leading_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + + +@pytest.mark.parametrize( + "operation,channel_axis,broken,error,message", + ( + ("for_leading_source_plane", 0, "domain", ValueError, "spatial_origin_yx"), + ("for_leading_source_plane", 0, "spacing", ValueError, "finite and positive"), + ( + "for_leading_source_plane", + 0, + "spacing-value", + AttributeError, + "with_missing_from", + ), + ("for_leading_source_plane", 0, "plane", ValueError, "invalid-axis"), + ("for_leading_source_plane", False, "domain", ValueError, "spatial_origin_yx"), + ( + "for_leading_source_plane", + False, + "spacing", + ValueError, + "finite and positive", + ), + ("for_leading_source_plane", False, "plane", TypeError, "must be int or None"), + ("for_source_plane", False, "domain", ValueError, "spatial_origin_yx"), + ("for_source_plane", False, "spacing", ValueError, "finite and positive"), + ( + "for_source_plane", + False, + "spacing-value", + AttributeError, + "with_missing_from", + ), + ("for_source_plane", False, "plane", TypeError, "must be int or None"), + ( + "without_leading_plane_axis", + 0, + "domain", + ValueError, + "both plane and channel", + ), + ( + "without_leading_plane_axis", + 0, + "spacing", + ValueError, + "both plane and channel", + ), + ( + "without_leading_plane_axis", + 0, + "spacing-value", + ValueError, + "both plane and channel", + ), + ( + "without_leading_plane_axis", + 0, + "plane", + ValueError, + "both plane and channel", + ), + ("without_leading_plane_axis", "1", "plane", TypeError, "not supported"), + ), +) +def test_projection_preserves_cross_invalid_constructor_guard_order( + operation, channel_axis, broken, error, message +): + metadata = ImagePayloadMetadata( + source_path="/tmp/source.tif", + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + ) + metadata.source_channel_axis = channel_axis + if broken == "domain": + metadata.source_spatial_domain = SourceSpatialDomain(origin_yx=(1,)) + elif broken == "spacing": + metadata.source_component_metadata = {"OpenHCSSourceVoxelSpacingZYX": "0,1,1"} + elif broken == "spacing-value": + metadata.source_voxel_spacing = None + else: + metadata.plane_axis = "invalid-axis" + with pytest.raises(error, match=message): + if operation == "without_leading_plane_axis": + metadata.without_leading_plane_axis() + else: + getattr(metadata, operation)(0) + + +def test_missing_leading_axis_precedes_plane_field_effects(): + metadata, events = _metadata_with_observed_plane_fields(proof_raises=True) + metadata.plane_axis = None + metadata.source_channel_axis = 0 + with pytest.raises(ValueError, match="requires a declared plane axis"): + metadata.for_leading_source_plane(0) + assert events == [] + + +def test_projection_spacing_failure_precedes_other_invalid_source_fields(): + metadata = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE) + metadata.source_component_metadata = {"OpenHCSSourceVoxelSpacingZYX": "0,1,1"} + metadata.source_spatial_domain = SourceSpatialDomain(origin_yx=(1,)) + metadata.source_channel_axis = False + metadata.plane_axis = "invalid-axis" + with pytest.raises(ValueError, match="finite and positive"): + metadata.for_leading_source_plane(0) + + +@pytest.mark.parametrize( + "operation,normalization_axes", + ( + ("without_leading_plane_axis", (RuntimePlaneAxis.RUNTIME_SLICE, None)), + ( + "for_leading_source_plane", + (RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.RUNTIME_SLICE, None), + ), + ), +) +def test_projection_normalizes_owned_metadata_at_each_historical_phase( + monkeypatch, operation, normalization_axes +): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, 2) + input_provenance = metadata.source_provenance + input_proof = metadata.unit_interval_intensity + births = [] + phases = [] + original_constructor = ImagePayloadMetadata.__post_init__ + original_normalize = ImagePayloadMetadata.normalize_metadata_fields + + def observe_constructor(self, *values): + births.append((self, self.plane_axis, self.source_channel_axis)) + original_constructor(self, *values) + + def observe_normalization(self): + phases.append((self, self.plane_axis, self.source_channel_axis)) + original_normalize(self) + + monkeypatch.setattr(ImagePayloadMetadata, "__post_init__", observe_constructor) + monkeypatch.setattr( + ImagePayloadMetadata, "normalize_metadata_fields", observe_normalization + ) + projected = ( + metadata.without_leading_plane_axis() + if operation == "without_leading_plane_axis" + else metadata.for_leading_source_plane(1) + ) + + assert births == [(projected, RuntimePlaneAxis.RUNTIME_SLICE, 2)] + assert tuple(axis for _, axis, _ in phases) == normalization_axes + assert all(owner is projected for owner, _, _ in phases) + assert tuple(channel for _, _, channel in phases) == (2,) * ( + len(normalization_axes) - 1 + ) + (1,) + assert projected is not metadata + assert metadata.source_provenance is input_provenance + assert metadata.unit_interval_intensity is input_proof + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_channel_axis == 2 + assert metadata.source_plane_intensity_scales == (255.0, 65535.0) + assert projected.source_provenance is not input_provenance + assert projected.unit_interval_intensity is not input_proof + + +@pytest.mark.parametrize("combined", (False, True)) +def test_projection_preserves_contributor_derivation_constructor_error_order( + monkeypatch, combined +): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + invalid_domain = SourceSpatialDomain(origin_yx=(1,)) + metadata.source_spatial_domain = invalid_domain + provenance = metadata.source_provenance + calls = [] + + def fail_contributors(self): + calls.append(self) + raise RuntimeError("contributor derivation observed") + + monkeypatch.setattr( + type(provenance), "with_runtime_planes_as_contributors", fail_contributors + ) + if combined: + with pytest.raises(ValueError, match="spatial_origin_yx"): + metadata.for_leading_source_plane(1) + assert calls == [] + else: + with pytest.raises(RuntimeError, match="contributor derivation observed"): + metadata.without_leading_plane_axis() + assert calls == [provenance] + assert metadata.source_provenance is provenance + assert metadata.source_spatial_domain is invalid_domain + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_plane_intensity_scales == (255.0, 65535.0) + + +@pytest.mark.parametrize("combined", (False, True)) +def test_failed_owned_axis_transform_does_not_mutate_source(monkeypatch, combined): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, 2) + provenance = metadata.source_provenance + proof = metadata.unit_interval_intensity + + def fail_proof(self): + raise RuntimeError("axis proof derivation observed") + + monkeypatch.setattr( + ImageUnitIntervalIntensityMetadata, "without_source_planes", fail_proof + ) + with pytest.raises(RuntimeError, match="axis proof derivation observed"): + if combined: + metadata.for_leading_source_plane(1) + else: + metadata.without_leading_plane_axis() + assert metadata.source_provenance is provenance + assert metadata.unit_interval_intensity is proof + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_channel_axis == 2 + assert metadata.source_plane_intensity_scales == (255.0, 65535.0) + assert metadata.source_plane_dtypes == ("uint8", "uint16") + + +@pytest.mark.parametrize( + "plane_index,expected_scale", + ((None, 255), (-1, 255), (0, 255), (1, 65535), (2, 255)), +) +def test_owned_proof_projection_preserves_scalar_fallback_and_axis_removal( + plane_index, expected_scale +): + proof = ImageUnitIntervalIntensityMetadata( + scale=255, source_plane_scales=(None, 65535) + ) + metadata = ImagePayloadMetadata(unit_interval_intensity=proof) + projected = metadata.project_intensity_proof(plane_index) + assert projected is not proof + assert projected.scale == expected_scale + assert projected.source_plane_scales == () + assert metadata.unit_interval_intensity is proof + assert proof.source_plane_scales == (None, 65535) + if plane_index is not None: + assert metadata.unit_interval_intensity_scale_for_source_plane(plane_index) == ( + expected_scale + ) + + +@pytest.mark.parametrize("plane_index", (None, 0, 1)) +def test_owned_proof_projection_observes_absence_and_live_replacement(plane_index): + metadata = ImagePayloadMetadata(intensity_scale=255) + assert metadata.project_intensity_proof(plane_index) is None + if plane_index is not None: + assert ( + metadata.unit_interval_intensity_scale_for_source_plane(plane_index) is None + ) + proof = ImageUnitIntervalIntensityMetadata(scale=7, source_plane_scales=(11, 13)) + metadata.unit_interval_intensity = proof + assert metadata.project_intensity_proof(plane_index).scale == ( + 7 if plane_index is None else (11, 13)[plane_index] + ) + metadata.unit_interval_intensity = None + assert metadata.project_intensity_proof(plane_index) is None + + +def test_quantization_queries_preserve_integer_conversion_and_fallback_identity(): + calls = [] + + class ObservedInt(int): + def __int__(self): + calls.append(self) + return super().__int__() + + scalar = ObservedInt(255) + plane = ObservedInt(65535) + metadata = ImagePayloadMetadata( + unit_interval_intensity=ImageUnitIntervalIntensityMetadata( + scale=scalar, source_plane_scales=(None, plane) + ) + ) + assert metadata.unit_interval_intensity_scale_for_source_plane(0) is scalar + assert metadata.unit_interval_intensity_scale_for_source_plane(-1) is scalar + assert calls == [] + assert metadata.unit_interval_intensity_scale_for_source_plane(1) == 65535 + assert calls == [plane] + calls.clear() + assert metadata.project_intensity_proof(1).scale == 65535 + assert calls == [plane] + assert metadata.project_intensity_proof(None).scale is scalar + assert calls == [plane] diff --git a/tests/unit/test_live_fractional_z_point_source.py b/tests/unit/test_live_fractional_z_point_source.py new file mode 100644 index 0000000000..eb2dfca0d7 --- /dev/null +++ b/tests/unit/test_live_fractional_z_point_source.py @@ -0,0 +1,229 @@ +"""Original point writer -> stream metadata -> unchanged projection controls. + +No compilation, viewer instance, native process or endpoint connection. +""" +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import load_rois_from_zip +from polystore.roi_converters import NapariROIConverter +from polystore.streaming.identity import StreamProducerIdentity +from polystore.streaming.viewer_transport import ViewerStreamProducer, ViewerStreamSourceIdentity +from polystore.streaming_constants import StreamingDataType +from zmqruntime.config import TransportMode +from zmqruntime.viewer_protocol import ViewerTransportEndpoint + +from openhcs.constants.constants import VariableComponents +from openhcs.core.config import NapariStreamingConfig +from openhcs.core.memory import numpy as numpy_func +from openhcs.core.measurement_row_materialization import MeasurementSparseColumnarRows +from openhcs.core.roi_point_metadata import ROIFractionalZ +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_measurements import ( + MeasurementScope, MeasurementSubject, MeasurementTable, ObjectCoreMeasurementFeature, +) +from openhcs.core.runtime_tabular_values import FieldSpec +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import ( + OriginalSourceMetadata, SourceFilterPathMetadata, SourceVoxelSpacing, +) +from openhcs.core.steps.function_artifact_materialization import ArtifactStreamSourceMetadataAuthority +from openhcs.core.steps.stream_component_semantics import StreamComponentMessageExtraAuthority +from openhcs.core.streaming_config_declarations import ViewerType +from openhcs.core.streaming_config_factory import StreamingViewerRuntimeConfig, StreamingViewerSurface +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.processing.materialization import ( + CsvOptions, MaterializationSpec, PointROIOptions, materialization_outputs, materialize, +) +from openhcs.processing.materialization.core import PointROIOutput, ViewerStreamBackendCallKwargs +from openhcs.runtime.napari_streaming_handlers import NapariStreamLayerAddress, NapariStreamLayerItem +from openhcs.runtime.napari_viewer_server import NapariPointsLayerDisplayHandler +from openhcs.runtime.viewer_component_system import ViewerComponentValueDomainPayload + + +class SyntheticMicroscope: + """Original filename parser only; no metadata or runtime service needed.""" + + parser = SourceSchemaFilenameParser() + + +def point_fixture(root: Path, planes: int, origin: int, *, source_records=False): + components = tuple( + dict(well='A01', site=1, channel=1, z_index=origin + index, timepoint=1) + for index in range(planes) + ) + paths = tuple(str(root / f'A01_s001_w1_z{origin + index:03d}_t001.tif') for index in range(planes)) + if source_records: + for component, path in zip(components, paths, strict=True): + OriginalSourceMetadata.from_mapping({ + 'well': 'A01', 'site': '001', 'channel': '1', + 'timepoint': '001', 'z_index': f"{component['z_index']:03d}", + }).merge_into(component, path=path) + SourceFilterPathMetadata.from_paths((Path(path).name, path)).merge_into( + component, path=path, + ) + SourceVoxelSpacing((2.0, 0.65, 0.65)).merge_into(component, path=path) + volume = np.zeros((planes, 5, 7), dtype=np.uint16) + volume[:, 1:3, 2:4] = 1 + for path, plane in zip(paths, volume): + tifffile.imwrite(path, plane) + provenance = SourceImageProvenancePlanes.from_components(paths=paths, component_metadata=components) + + @numpy_func(contract=ProcessingContract.PURE_3D) + def synthetic_centres(image: np.ndarray) -> MeasurementTable: + z, y, x = np.argwhere(image > 0).mean(axis=0) + return MeasurementTable( + name='synthetic_centres', + rows=MeasurementSparseColumnarRows.from_rows( + (dict(object_label=7, center_z=float(z), center_y=float(y), center_x=float(x)),), + fields=(FieldSpec('object_label', int), FieldSpec('center_z', float), + FieldSpec('center_y', float), FieldSpec('center_x', float)), + ), + source_image_provenance_planes=provenance, + subject=MeasurementSubject(MeasurementScope.OBJECT, 'synthetic', 'object_label'), + ) + + return synthetic_centres(volume), components, paths + + +@pytest.mark.parametrize('planes,origin', ((1, 0), (4, 0), (4, 10))) +@pytest.mark.parametrize('selected', (False, True)) +@pytest.mark.parametrize('source_records', (False, True)) +def test_original_point_writer_live_domain_anchor_and_reopen(tmp_path, planes, origin, selected, source_records): + table, components, paths = point_fixture(tmp_path, planes, origin, source_records=source_records) + feature = ObjectCoreMeasurementFeature + options = PointROIOptions( + z_feature=feature.CENTER_Z, y_feature=feature.CENTER_Y, x_feature=feature.CENTER_X, + source='centres' if selected else None, + ) + payload = {'centres': table} if selected else table + spec = MaterializationSpec(CsvOptions(source=options.source), options) + fm = FileManager({'disk': DiskStorageBackend()}) + outputs = materialization_outputs( + spec, payload, str(tmp_path / 'centres'), fm, + variable_components=(VariableComponents.Z_INDEX,), + ) + point_output, = (output for output in outputs if isinstance(output, PointROIOutput)) + anchor = point_output.source_identity + assert anchor.path == paths[0] + assert dict(anchor.component_metadata) == components[0] + assert point_output.metadata.source_image_provenance_planes.count == planes + assert replace(point_output).source_identity == anchor + assert not spec.emits_variable_component_planes(payload) + identities = spec.stream_source_identities(payload) + assert tuple(identity.path for identity in identities) == paths + assert tuple(dict(identity.component_metadata) for identity in identities) == components + + archive = Path(materialize( + MaterializationSpec(options), data=payload, path=str(tmp_path / 'saved'), + filemanager=fm, backends=['disk'], backend_kwargs={}, + )) + rois = load_rois_from_zip(archive) + saved_metadata = ROIArchiveSourceMetadata.decode(rois) + saved_domain = ROIFractionalZ.source_component_domain(rois, saved_metadata) + assert tuple(dict(item) for item in saved_domain) == components + assert saved_metadata.source_image_provenance_planes == point_output.metadata.source_image_provenance_planes + if planes > 1: + assert 'z_index' not in (saved_metadata.source_component_metadata or {}) + + surface = StreamingViewerSurface( + runtime_config=StreamingViewerRuntimeConfig( + transport_endpoint=ViewerTransportEndpoint(host='127.0.0.1', port=6199, transport_mode=TransportMode.TCP), + persistent=True, viewer_type=ViewerType.NAPARI, + ), + display_config=NapariStreamingConfig(enabled=True), + source=ViewerStreamSourceIdentity(microscope_handler=SyntheticMicroscope(), plate_path=None), + ) + items = ArtifactStreamSourceMetadataAuthority.metadata_items( + materialization_spec=spec, data=payload, fallback_source_identity=None, + ) + authority = StreamComponentMessageExtraAuthority.from_viewer_surface(surface, source_metadata_items=items) + assert 'z_index' in authority.layout.component_order + producer = StreamProducerIdentity.pipeline_output( + output_kind='artifact', output_key='synthetic_centres', projection_key='synthetic_centres', + step_name='Own engineering only', pipeline_position=0, artifact_kind='roi', + ) + kwargs = ViewerStreamBackendCallKwargs(authority.viewer_backend_kwargs( + producer=ViewerStreamProducer.from_identity(producer), + )).to_filemanager_kwargs(point_output) + request = kwargs['stream_request'] + streamed = request.source.metadata.component_metadata_for_item(index=0, file_path=point_output.path) + assert int(streamed['z_index']) == origin + domain = ViewerComponentValueDomainPayload.from_wire_mapping( + authority.payload().to_wire_mapping()['component_value_domain'], context='own point source fixture', + ) + item = NapariStreamLayerItem( + data=NapariROIConverter.rois_to_shapes(rois), producer=producer, + address=NapariStreamLayerAddress(components=anchor.component_metadata, path=str(archive), stream_layer_data_type=StreamingDataType.POINTS), + image_metadata=saved_metadata, plane_component_domain=domain, + ) + handler = NapariPointsLayerDisplayHandler() + occupied = handler.geometric_component_values([item], authority.component_axis_semantics) + assert occupied == {'z_index': list(range(origin, origin + planes))} + with pytest.raises(ValueError, match='exceeds the declared Z domain'): + handler.geometric_component_values( + [replace(item, data=NapariROIConverter.rois_to_shapes([ + ROIFractionalZ(float(planes)).bind(rois[0]), + ]))], authority.component_axis_semantics, + ) + + +def test_point_source_admission_reuses_original_typed_payload_contract(): + feature = ObjectCoreMeasurementFeature + options = PointROIOptions(z_feature=feature.CENTER_Z, y_feature=feature.CENTER_Y, x_feature=feature.CENTER_X) + with pytest.raises(TypeError, match='requires a MeasurementTable'): + options.stream_source_identities(np.zeros((1, 5, 7))) + + +@pytest.mark.parametrize('field,value,error', ( + ('well', 'B01', 'vary outside Z'), + ('site', 2, 'vary outside Z'), + ('channel', 2, 'vary outside Z'), + ('timepoint', 2, 'vary outside Z'), + ('well', None, 'inconsistent components'), + ('site', None, 'inconsistent components'), + ('channel', None, 'inconsistent components'), + ('timepoint', None, 'inconsistent components'), + ('z_index', 0, 'consecutive and ordered'), + ('z_index', 3, 'consecutive and ordered'), + ('z_index', None, 'require z_index'), + ('z_index', True, 'must be integers'), + ('z_index', 1.5, 'must be integers'), + ('z_index', '1.5', 'must be integers'), + ('OpenHCSSourceVoxelSpacingZYX', '2,0.7,0.7', 'inconsistent calibration'), + ('OpenHCSSourceVoxelSpacingUnit', 'relative', 'inconsistent calibration'), +)) +def test_point_domain_preserves_real_coordinate_and_calibration_rejections(tmp_path, field, value, error): + table, components, paths = point_fixture(tmp_path, 4, 0, source_records=True) + invalid = tuple(dict(plane) for plane in components) + if value is None: + del invalid[1][field] + else: + invalid[1][field] = value + metadata = ImagePayloadMetadata( + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=paths, component_metadata=invalid, + ), + ) + from polystore.roi import ROI, PointShape + rois = [ROIFractionalZ(1.5).bind(ROI(shapes=[PointShape(y=1.5, x=2.5)]))] + with pytest.raises(ValueError, match=error): + ROIFractionalZ.source_component_domain(rois, metadata) + + +@pytest.mark.parametrize('coordinate', (-0.5, 3.5)) +def test_point_domain_preserves_fractional_z_bounds_with_literal_provenance(tmp_path, coordinate): + table, components, paths = point_fixture(tmp_path, 4, 0, source_records=True) + from polystore.roi import ROI, PointShape + rois = [ROIFractionalZ(coordinate).bind(ROI(shapes=[PointShape(y=1.5, x=2.5)]))] + with pytest.raises(ValueError, match='outside its source planes'): + ROIFractionalZ.source_component_domain( + rois, ImagePayloadMetadata(source_provenance=table.source_provenance), + ) diff --git a/tests/unit/test_materialization_batch_outcomes.py b/tests/unit/test_materialization_batch_outcomes.py new file mode 100644 index 0000000000..a24b092c58 --- /dev/null +++ b/tests/unit/test_materialization_batch_outcomes.py @@ -0,0 +1,232 @@ +"""Actual batch outcomes, independent of subsequent logical-value rendering.""" + +from dataclasses import replace +from pathlib import Path +import pickle +from types import MappingProxyType + +import pytest +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.memory import MemoryStorageBackend + +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.compiled_step_plan import CompiledStepPlan +from openhcs.core.debug import NoOpDebugExecutionPolicy +from openhcs.core.function_patterns import compile_function_pattern +from openhcs.core.orchestrator.compiled_plate_execution import ( + CompiledPlateExecutionExtras, + CompiledPlateExecutionResults, +) +from openhcs.core.orchestrator.execution_result import ( + ExecutionResult, + RuntimeContextObservation, + RuntimeExecutionObservation, + RuntimeObservationMode, +) +from openhcs.core.orchestrator.worker_execution import ( + _execute_axis_with_sequential_combinations, +) +from openhcs.core.orchestrator.worker_lanes import WorkerLaneExecutionContext +from openhcs.core.runtime_exports import RuntimeExportObservation +from openhcs.core.steps.abstract import StepExecutionObservation +from openhcs.core.steps.function_step import FunctionStep +from openhcs.processing.materialization import ( + CsvOptions, + JsonOptions, + MaterializationSpec, + SavedMaterializationOutputs, + prepare_materialization, +) +from openhcs.processing.materialization.core import _WRITERS_BY_OPTIONS + + +class CsvOnlyMemoryBackend(MemoryStorageBackend): + def accepts_payload(self, data, path): + return Path(path).suffix == ".csv" + + +def test_rendered_batch_reuses_exact_outputs_after_logical_payload_mutates( + tmp_path, monkeypatch +): + manager = FileManager({"disk": DiskStorageBackend()}) + rows = [{"Area": 42}] + batch = prepare_materialization( + MaterializationSpec(CsvOptions(), JsonOptions()), + rows, + str(tmp_path / "measurements"), + manager, + ("disk",), + ) + original_outputs = batch.outputs + rows[0]["Area"] = 999 + + def render_again(*_args): + pytest.fail( + "Saving a prepared batch must not render mutable logical input again" + ) + + for option_type in (CsvOptions, JsonOptions): + monkeypatch.setitem( + _WRITERS_BY_OPTIONS, + option_type, + replace(_WRITERS_BY_OPTIONS[option_type], write=render_again), + ) + outcome = batch.save() + + assert isinstance(outcome, SavedMaterializationOutputs) + assert outcome.outputs_for_backend("disk") == original_outputs + assert all( + left is right + for left, right in zip( + outcome.outputs_for_backend("disk"), original_outputs, strict=True + ) + ) + assert Path(batch.primary_path).read_text().splitlines() == ["Area", "42"] + assert Path(original_outputs[1].path).read_text().find("999") == -1 + + +def test_saved_outcomes_include_only_backend_accepted_outputs(tmp_path): + manager = FileManager({"memory": CsvOnlyMemoryBackend()}) + batch = prepare_materialization( + MaterializationSpec(CsvOptions(), JsonOptions()), + [{"Area": 42}], + str(tmp_path / "measurements"), + manager, + ("memory",), + ) + + outcome = batch.save() + + assert outcome.outputs_for_backend("memory") == (batch.outputs[0],) + assert outcome.outputs_for_backend("disk") == () + assert manager.exists(batch.outputs[0].path, "memory") + assert not manager.exists(batch.outputs[1].path, "memory") + + +def test_failed_save_does_not_return_a_successful_outcome(tmp_path, monkeypatch): + manager = FileManager({"disk": DiskStorageBackend()}) + batch = prepare_materialization( + MaterializationSpec(CsvOptions()), + [{"Area": 42}], + str(tmp_path / "measurements"), + manager, + ("disk",), + ) + + def failed_save(*_args, **_kwargs): + raise OSError("destination is unavailable") + + monkeypatch.setattr(manager, "save_batch", failed_save) + with pytest.raises(OSError, match="destination is unavailable"): + batch.save() + assert not Path(batch.primary_path).exists() + + +class EmittedOutputStep(FunctionStep): + def __init__(self, path): + super().__init__(lambda image: image) + self.path = path + + def process(self, context, step_index): + self.path.write_text("Area\n42\n") + return StepExecutionObservation(MappingProxyType({}), (self.path,)) + + +class FrozenWorkerContext(ProcessingContext): + def __init__(self): + super().__init__( + axis_id="A01", + filemanager=FileManager({"memory": MemoryStorageBackend()}), + step_plans={ + 0: CompiledStepPlan( + step_index=0, + step_name="Save", + step_type="FunctionStep", + axis_id="A01", + compiled_function_pattern=compile_function_pattern( + lambda image: image, {}, {} + ), + ) + }, + ) + self.freeze() + self.released = False + + def release_execution_image_cache(self): + super().release_execution_image_cache() + self.released = True + + +@pytest.mark.parametrize("mode", tuple(RuntimeObservationMode)) +def test_worker_exports_actual_step_outcomes_after_resource_release( + tmp_path, monkeypatch, mode +): + import openhcs.core.orchestrator.worker_execution as worker + + path = tmp_path / "actual.csv" + context = FrozenWorkerContext() + lane = WorkerLaneExecutionContext( + execution_id="execution", + plate_id="plate", + debug_execution_policy=NoOpDebugExecutionPolicy(), + worker_slot="worker-0", + worker_assignments={"worker-0": ("A01",)}, + ) + monkeypatch.setattr(worker, "emit", lambda **_kwargs: None) + + def forbidden_reconstruction(*_args, **_kwargs): + pytest.fail("Ordinary execution must consume actual step outcomes") + + monkeypatch.setattr( + worker, + "preview_reused_materialized_artifact_locations", + forbidden_reconstruction, + ) + monkeypatch.setattr( + worker, + "preview_reused_runtime_export_paths", + forbidden_reconstruction, + ) + result = _execute_axis_with_sequential_combinations( + [EmittedOutputStep(path)], + [("A01", context)], + lane, + mode, + release_axis_resources=False, + ) + transported = pickle.loads(pickle.dumps(result)) + + assert context.released + assert transported.is_success() + assert transported.runtime_observation.contexts[0].runtime_export_paths == (path,) + assert transported.runtime_observation.contexts[0].records == () + assert RuntimeExportObservation.from_runtime_observations( + (transported.runtime_observation,) + ).table_outputs == (path,) + + +def test_parent_and_worker_exports_share_existing_observation_authority(tmp_path): + worker_path = tmp_path / "worker.csv" + plate_path = tmp_path / "plate.csv" + unrelated = tmp_path / "unrelated.csv" + for path in (worker_path, plate_path, unrelated): + path.write_text("Area\n42\n") + worker = RuntimeExecutionObservation( + contexts=(RuntimeContextObservation("A01", (), (worker_path,)),) + ) + parent = RuntimeExecutionObservation( + contexts=(RuntimeContextObservation("A01", (), (plate_path,)),) + ) + results = CompiledPlateExecutionResults( + {"A01": ExecutionResult.success("A01", runtime_observation=worker)}, + extras=CompiledPlateExecutionExtras( + viewer_states_by_port={}, runtime_observation=parent + ), + ) + exports = RuntimeExportObservation.from_runtime_observations( + results.runtime_observations + ) + + assert exports.output_files == (worker_path, plate_path) + assert unrelated not in exports.output_files diff --git a/tests/unit/test_materialization_core.py b/tests/unit/test_materialization_core.py index 39022ad340..9207d15d7f 100644 --- a/tests/unit/test_materialization_core.py +++ b/tests/unit/test_materialization_core.py @@ -2394,7 +2394,7 @@ def test_roi_materialization_spec_reports_projected_source_identities() -> None: assert spec.emits_variable_component_planes(payload) is True assert [ identity.component_metadata - for identity in spec.emitted_source_identities(payload) + for identity in spec.stream_source_identities(payload) ] == [ { "well": "A01", diff --git a/tests/unit/test_napari_streaming_handlers.py b/tests/unit/test_napari_streaming_handlers.py index bb9d296666..4e3a241559 100644 --- a/tests/unit/test_napari_streaming_handlers.py +++ b/tests/unit/test_napari_streaming_handlers.py @@ -6251,6 +6251,92 @@ def test_napari_points_layer_uses_exact_fractional_z_from_native_roi_metadata(): assert tuple(server.viewer.calls[-1][1][0]) == (2.375, 1.25, 3.5) +@pytest.mark.parametrize( + "values,unit,expected_scale,expected_units", + ( + ((2.0, 0.65, 0.65), "micrometers", (1.0, 2.0, 0.65, 0.65), + ("dimensionless", "micrometer", "micrometer", "micrometer")), + ((2.0, 1.0, 1.0), "relative", (1.0, 2.0, 1.0, 1.0), + ("dimensionless", "dimensionless", "dimensionless", "dimensionless")), + ((0.65, 0.65), "micrometers", (1.0, 1.0, 0.65, 0.65), + ("dimensionless", "dimensionless", "micrometer", "micrometer")), + ((), "micrometers", (1.0, 1.0, 1.0, 1.0), + ("dimensionless", "dimensionless", "pixel", "pixel")), + ), +) +def test_spatial_presentation_owns_scale_and_world_translation( + values, unit, expected_scale, expected_units +): + from openhcs.core.source_metadata import SourceVoxelSpacing, SourceVoxelSpacingUnit + + presentation = _axis_presentation( + layer_key="own-calibrated-source", + projected_axis_components=("channel", "z_index"), + component_values={"channel": [4], "z_index": [2, 3, 4, 5]}, + axis_offsets=(3, 2), + ) + item = _layer_item( + {"channel": 4, "z_index": 2}, np.zeros((5, 7), dtype=np.uint16) + ) + item = replace(item, image_metadata=ImagePayloadMetadata( + source_voxel_spacing=SourceVoxelSpacing(values, SourceVoxelSpacingUnit(unit)) + )) + kwargs = presentation.spatial_layer_kwargs([item]) + assert kwargs["scale"] == expected_scale + assert kwargs["units"] == expected_units + assert kwargs["translate"] == (3.0, 2 * expected_scale[1], 0.0, 0.0) + payload_kwargs = presentation.spatial_layer_kwargs([item], ("band",)) + assert payload_kwargs["scale"] == (*expected_scale[:2], 1.0, *expected_scale[2:]) + assert payload_kwargs["units"] == (*expected_units[:2], "dimensionless", *expected_units[2:]) + + +def test_calibrated_native_image_point_and_semantic_navigation_agree_without_qt(): + from napari.layers import Image, Points + from openhcs.core.source_metadata import SourceVoxelSpacing + + presentation = _axis_presentation( + layer_key="own-calibrated-source", + projected_axis_components=("z_index",), + component_values={"z_index": [0, 1, 2, 3]}, + ) + item = replace( + _layer_item({"z_index": 0}, np.zeros((5, 7), dtype=np.uint16)), + image_metadata=ImagePayloadMetadata( + source_voxel_spacing=SourceVoxelSpacing((2.0, 0.65, 0.65)) + ), + ) + kwargs = presentation.spatial_layer_kwargs([item]) + pixels = np.zeros((4, 5, 7), dtype=np.uint16) + point = np.array([[1.5, 1.5, 2.5]]) + raw = Image(pixels, **kwargs) + centres = Points(point, **kwargs) + assert raw.data is pixels + np.testing.assert_array_equal(centres.data, point) + assert raw.data_to_world((1.5, 1.5, 2.5)) == centres.data_to_world(point[0]) + np.testing.assert_allclose(centres.data_to_world(point[0]), (3.0, 0.975, 1.625)) + dims = Dims(ndim=3, range=((0, 6, 2), (0, 2.6, 0.65), (0, 3.9, 0.65))) + dims.current_step = (presentation.viewer_step(2, 0), 0, 0) + assert dims.point[0] == 4.0 + assert raw.world_to_data(dims.point)[0] == 2.0 + state = NapariDimensionLayerState( + labels={"z_index": ["Z0", "Z1", "Z2", "Z3"]}, + presentation=presentation, + ) + assert state.label_parts_for_current_step(dims.current_step) == ("Z2",) + + +def test_shape_features_exclude_source_transport_without_mutating_payload(): + from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata + + full_metadata = {"label": 7, "response": 0.25, + ROIArchiveSourceMetadata.FIELD: {"source_provenance": "retained"}} + features = NapariShapeFeatureColumns() + features.append(full_metadata, label=7, path="/own/synthetic.roi.zip") + assert ROIArchiveSourceMetadata.FIELD in full_metadata + assert ROIArchiveSourceMetadata.FIELD not in features.values + assert features.values["response"] == [0.25] + + def test_napari_fractional_z_points_are_relative_to_the_declared_anchor(): napari_viewer_server = pytest.importorskip("openhcs.runtime.napari_viewer_server") server = _FakeNapariServer() diff --git a/tests/unit/test_opaque_main_flow_image_domain.py b/tests/unit/test_opaque_main_flow_image_domain.py new file mode 100644 index 0000000000..6553bf96ad --- /dev/null +++ b/tests/unit/test_opaque_main_flow_image_domain.py @@ -0,0 +1,445 @@ +"""Whole image domains survive the producer/cache/next-load handoff.""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pytest + +from openhcs.constants.constants import MEMORY_TYPE_NUMPY +from openhcs.core.compiled_step_plan import CompiledStepPlan +from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.function_patterns import compile_function_pattern +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.runtime_slice_projection import RuntimeSliceProjection +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.core.steps.function_runtime import PatternGroupData, PatternGroupRuntime +from openhcs.core.aligned_image_payload import AlignedImageStack, ImageOutputBundle, AlignedImageSliceContext +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.steps.function_output_manifest import step_output_manifest, StepOutputManifestStore +from openhcs.core.source_bindings import CompiledSourceBindingPlan, SourceBindingRuntimeContext +from openhcs.core.steps.function_runtime import SourceBindingRuntimeContextRequest +from openhcs.core.step_dependencies import StepInputDependency +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes + + +def _identity(image): + return image + + +def _runtime(): + pattern = compile_function_pattern(_identity, {}, {}) + runtime = object.__new__(PatternGroupRuntime) + runtime.request = SimpleNamespace( + compiled_group=pattern.default_group, + execution_plan=CompiledStepPlan( + step_index=0, + step_name="WholeImage", + step_type="FunctionStep", + axis_id="A01", + output_memory_type=MEMORY_TYPE_NUMPY, + input_memory_type=MEMORY_TYPE_NUMPY, + variable_components=(), + artifact_inputs={}, + artifact_outputs={}, + execution_group_scope=ComponentGroupScope.ungrouped(), + ), + component_key=None, + ) + return runtime + + +@pytest.mark.parametrize( + ("shape", "channel_axis"), + [((4, 5), None), ((4, 5, 3), -1), ((3, 4, 5), None), ((1, 4, 5), None)], + ids=("plane", "rgb", "volume", "one-z-volume"), +) +def test_opaque_whole_image_is_copied_without_a_new_execution_axis(shape, channel_axis): + pixels = np.arange(np.prod(shape), dtype=np.float32).reshape(shape) + mask_shape = shape[:2] if channel_axis is not None else shape + mask = np.ones(mask_shape, dtype=bool) + metadata = ImagePayloadMetadata( + source_path="/input/A01_s001_w1_z001_t001.tif", + source_component_metadata={"well": "A01", "site": "1", "channel": "1"}, + source_image_names=("WholeImage",), + source_channel_axis=channel_axis, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(12, 13), + ), + ) + payload = metadata.payload_with(pixels, mask) + output = _runtime()._validate_and_unstack( + payload, + PatternGroupData( + matching_files=["/input/A01_s001_w1_z001_t001.tif", "/input/A01_s001_w1_z002_t001.tif"], + main_data_stack=np.zeros((2, 4, 5), dtype=np.float32), + ), + ) + assert len(output.slices) == 1 + assert output.slices[0] is payload + cached = output.stack_payload + assert image_payload_data(cached).shape == shape + assert image_payload_metadata(cached) == metadata + np.testing.assert_array_equal(image_payload_data(cached), pixels) + np.testing.assert_array_equal(image_payload_mask(cached), mask) + assert not np.shares_memory(image_payload_data(cached), pixels) + assert not np.shares_memory(image_payload_mask(cached), mask) + assert image_payload_metadata(cached) is not metadata + assert RuntimeSliceProjection.full_stack_value(cached) is cached + image_payload_data(cached).flat[0] = -100 + image_payload_mask(cached).flat[0] = False + assert pixels.flat[0] == 0 + assert mask.flat[0] + image_payload_metadata(cached).source_channel_axis = 0 + assert metadata.source_channel_axis == channel_axis + + +@pytest.mark.parametrize("count", [1, 3]) +def test_declared_runtime_axis_retains_its_exact_count(count): + pixels = np.arange(count * 4 * 5, dtype=np.float32).reshape(count, 4, 5) + payload = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE).payload_with(pixels) + output = _runtime()._validate_and_unstack( + payload, + PatternGroupData(matching_files=["source-1.tif", "source-2.tif"], main_data_stack=pixels), + ) + assert len(output.slices) == count + assert output.stack_payload is payload + assert image_payload_data(output.stack_payload).shape == (count, 4, 5) + assert image_payload_metadata(output.stack_payload).plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + + +@pytest.mark.parametrize("named", [False, True]) +@pytest.mark.parametrize("axis", [None, RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.SOURCE_BINDING]) +def test_nominal_alignment_owner_preserves_single_output_topology(named, axis): + shape = (3, 4, 5) if axis is None else (1, 4, 5) + payload = ImagePayloadMetadata(plane_axis=axis).payload_with(np.ones(shape, dtype=np.float32)) + context = AlignedImageSliceContext.main_flow("Image") + value = (ImageOutputBundle if named else AlignedImageStack)((payload,), (context,)) + output = _runtime()._validate_and_unstack( + value, PatternGroupData(matching_files=["source.tif"], main_data_stack=payload), + ) + if named and axis is not RuntimePlaneAxis.RUNTIME_SLICE: + expected_shape, expected_axis = shape, axis + else: + expected_shape = (1, 4, 5) if axis is RuntimePlaneAxis.RUNTIME_SLICE else (1, *shape) + expected_axis = RuntimePlaneAxis.RUNTIME_SLICE + assert image_payload_data(output.stack_payload).shape == expected_shape + assert image_payload_metadata(output.stack_payload).plane_axis is expected_axis + + +@pytest.mark.parametrize("count", [1, 3]) +def test_raw_array_fallback_keeps_explicit_runtime_unstack_contract(count): + runtime = _runtime() + runtime.request.context = ProcessingContext(axis_id="A01") + runtime.request.context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + pixels = np.arange(count * 4 * 5, dtype=np.float32).reshape(count, 4, 5) + output = runtime._validate_and_unstack( + pixels, PatternGroupData(matching_files=[f"source-{i}.tif" for i in range(count)], main_data_stack=pixels), + ) + cached = output.cache_payload_for_outputs(output.slices, [image_payload_metadata(p) for p in output.slices]) + assert image_payload_data(cached).shape == (count, 4, 5) + assert image_payload_metadata(cached).plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + + +class _MemoryFiles: + def __init__(self): + self.values = {} + + def exists(self, path, backend): + return path in self.values or any(value_path.startswith(path.rstrip("/") + "/") for value_path in self.values) + + def ensure_directory(self, path, backend): + pass + + def save_batch(self, payloads, paths, backend): + self.values.update(zip(paths, payloads, strict=True)) + + def load_batch(self, paths, backend): + return [self.values[path] for path in paths] + + def delete(self, path, backend): + del self.values[path] + + +@pytest.mark.parametrize("cache_hit", [False, True]) +@pytest.mark.parametrize("axis", [None, RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.SOURCE_BINDING]) +@pytest.mark.parametrize("shape", [(4, 5), (4, 5, 3), (3, 4, 5), (1, 4, 5)]) +def test_save_and_next_load_preserve_domain_and_independent_cache(tmp_path, monkeypatch, cache_hit, axis, shape, raw_named=False): + pixels = np.arange(np.prod(shape), dtype=np.float32).reshape(shape) + channel_axis = -1 if shape == (4, 5, 3) else None + if axis is not None: + pixels = pixels[None] + channel_axis = 3 if channel_axis is not None else None + mask = np.ones(pixels.shape[:-1] if channel_axis is not None else pixels.shape, dtype=bool) + path = "/input/A01_s001_w1_z001_t001.tif" + components = {"well": "A01", "site": "1", "channel": "1", "z_index": "1", "timepoint": "1"} + metadata = ImagePayloadMetadata( + source_path=path, source_component_metadata=components, source_image_names=("WholeImage",), + source_channel_axis=channel_axis, plane_axis=axis, + source_image_provenance_planes=( + SourceImageProvenancePlanes.from_components(paths=(path,), component_metadata=(components,)) + if axis is not None else SourceImageProvenancePlanes() + ), + ) + payload = pixels if raw_named else metadata.payload_with(pixels, mask) + if raw_named: + mask = None + producer = _runtime() + pattern = compile_function_pattern(_identity, {}, {}) + plan = replace(producer.request.execution_plan, output_dir=tmp_path, step_scope_id="whole-producer", compiled_function_pattern=pattern) + producer.request.execution_plan = plan + files = _MemoryFiles() + context = ProcessingContext(axis_id="A01", filemanager=files) + context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + producer.request.context = context + producer.pattern_repr = "whole" + output = producer._validate_and_unstack( + ImageOutputBundle((payload,), (AlignedImageSliceContext.main_flow("WholeImage"),)) if axis is not None or raw_named else payload, + PatternGroupData(matching_files=[path, "/input/A01_s001_w1_z002_t001.tif"], main_data_stack=pixels), + ) + records = producer._save_outputs(output, [path]) + manifest = step_output_manifest(context) + manifest.record_outputs(plan, records) + assert records[0].main_flow_plane_axis is axis + physical = files.values[records[0].output_path] + assert np.shares_memory(image_payload_data(physical), pixels) + if not cache_hit: + context.runtime_image_stack_cache.clear() + consumer_plan = replace( + plan, step_index=1, step_scope_id="whole-consumer", input_dir=tmp_path, + main_input_dependency=StepInputDependency.step_output(source_step_index=0, source_step_scope_id=plan.step_scope_id), + ) + consumer = PatternGroupRuntime(SimpleNamespace( + context=context, execution_plan=consumer_plan, compiled_group=pattern.default_group, + source_binding_plan=CompiledSourceBindingPlan.empty(), component_key=None, + pattern_group_info="A01_s001_w1_z{iii}_t001.tif", + )) + monkeypatch.setattr(consumer, "source_workspace_projection_authority", lambda: SimpleNamespace(projection_if_available=lambda: None)) + monkeypatch.setattr(SourceBindingRuntimeContextRequest, "from_context", classmethod(lambda cls, **kwargs: SimpleNamespace(runtime_context=SourceBindingRuntimeContext.empty))) + loaded = consumer._load_input_stack().main_data_stack + assert image_payload_data(loaded).shape == pixels.shape + assert image_payload_metadata(loaded).plane_axis is axis + np.testing.assert_array_equal(image_payload_data(loaded), pixels) + np.testing.assert_array_equal(image_payload_mask(loaded), mask) + assert not np.shares_memory(image_payload_data(loaded), pixels) + if mask is not None: + assert not np.shares_memory(image_payload_mask(loaded), mask) + assert image_payload_metadata(loaded) is not image_payload_metadata(physical) + image_payload_metadata(loaded).source_provenance = image_payload_metadata(loaded).source_provenance.with_source_image_names(("Changed",)) + assert image_payload_metadata(physical).source_image_names == ("WholeImage",) + image_payload_data(loaded).flat[0] = -100 + if mask is not None: + image_payload_mask(loaded).flat[0] = False + assert mask.flat[0] + assert pixels.flat[0] == 0 + consumer._record_main_flow_passthrough([records[0].relative_output_path]) + (passed,) = manifest.produced_records_for(consumer_plan) + assert passed.main_flow_plane_axis is axis + assert passed.image_metadata is records[0].image_metadata + + +@pytest.mark.parametrize("runtime_count", [1, 3]) +def test_mixed_named_outputs_retain_each_original_axis_before_projection(runtime_count): + contexts = (AlignedImageSliceContext.main_flow("Planes"), AlignedImageSliceContext.main_flow("Volume")) + planes = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE).payload_with(np.ones((runtime_count, 4, 5), dtype=np.float32)) + volume = ImagePayloadMetadata().payload_with(np.ones((4, 5) if runtime_count == 1 else (2, 4, 5), dtype=np.float32)) + bundle = ImageOutputBundle((planes, volume), contexts) + output = _runtime()._validate_and_unstack(bundle, PatternGroupData(matching_files=[], main_data_stack=planes)) + assert output.stack_payload is None + assert [output.plane_axis_for_output_context(c) for c in output.slice_contexts] == [RuntimePlaneAxis.RUNTIME_SLICE] * runtime_count + [None] + assert output.main_flow_source is bundle + + +def test_passed_through_record_preserves_collapsed_and_filename_coordinates(tmp_path): + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + record = ProducedOutputSemantics.from_output( + plan, tmp_path / "A01_s001_w1_z001_t001.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, filename_component_values={"well": "A01", "z_index": 1}, extension=".tif", source="test"), + main_flow_plane_axis=None, + ) + passed = record.passed_through(replace(plan, step_index=1, step_scope_id="next")) + assert passed.component_values == record.component_values + assert passed.filename_component_values == record.filename_component_values + assert passed.main_flow_plane_axis is None + + +def test_duplicate_physical_path_rejects_contradictory_whole_axis(tmp_path): + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + opaque = ProducedOutputSemantics.from_output( + plan, tmp_path / "A01_s001_w1_z001_t001.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, extension=".tif", source="test"), + main_flow_plane_axis=None, + ) + with pytest.raises(ValueError, match="conflicting main-flow image axes"): + StepOutputManifestStore._unique_output_path_records((opaque, replace(opaque, main_flow_plane_axis=RuntimePlaneAxis.RUNTIME_SLICE))) + + +@pytest.mark.parametrize("cache_hit", [False, True]) +@pytest.mark.parametrize("shape", [(4, 5), (4, 5, 3), (3, 4, 5), (1, 4, 5)]) +def test_named_raw_image_actual_save_and_reload_retains_opaque_domain(tmp_path, monkeypatch, cache_hit, shape): + test_save_and_next_load_preserve_domain_and_independent_cache( + tmp_path, monkeypatch, cache_hit, None, shape, raw_named=True, + ) + + +def test_record_axis_survives_replace_and_pickle(tmp_path): + import pickle + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + for axis in (None, RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.SOURCE_BINDING): + record = ProducedOutputSemantics.from_output( + plan, tmp_path / "A01_s001_w1_z001_t001.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, extension=".tif", source="test"), + main_flow_plane_axis=axis, + ) + assert replace(record).main_flow_plane_axis is axis + restored = pickle.loads(pickle.dumps(record)) + assert restored == record + assert restored.main_flow_plane_axis is axis + + +def _selected_pattern(name): + from openhcs.core.artifacts import ArtifactSpec, ImageArtifactType + from openhcs.core.pipeline.function_contracts import artifact_inputs + from openhcs.core.function_patterns import InvocationArtifactInputEdgePlan, InvocationArtifactInputProjectionKey + spec = ArtifactSpec.input(name, ImageArtifactType) + + @artifact_inputs(spec) + def consume(image): + return image + + pattern = compile_function_pattern(consume, {}, {}) + invocation = pattern.default_group.invocations[0] + invocation = invocation.with_artifact_input_edges((InvocationArtifactInputEdgePlan( + key=InvocationArtifactInputProjectionKey(invocation_key=invocation.key, input_index=0), + spec=spec, storage_plan=None, projection=None, consumes_main_flow=True, + ),)) + return replace(pattern, groups=(replace(pattern.default_group, invocations=(invocation,)),)) + + +@pytest.mark.parametrize("cache_hit", [False, True]) +def test_saved_mixed_named_cohort_rejects_joint_load_and_preserves_selected_domains(tmp_path, monkeypatch, cache_hit): + from openhcs.core.artifacts import ImageArtifactType + source = "/input/A01_s001_w1_z001_t001.tif" + components = {"well": "A01", "site": "1", "channel": "1", "z_index": "1", "timepoint": "1"} + contexts = tuple(AlignedImageSliceContext.main_flow(name, artifact_kind=ImageArtifactType.value) for name in ("Planes", "Volume")) + scalar_metadata = ImagePayloadMetadata(source_path=source, source_component_metadata=components) + declared_metadata = scalar_metadata.replace_fields( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components(paths=(source,), component_metadata=(components,)), + ) + planes = declared_metadata.payload_with(np.arange(20, dtype=np.float32).reshape(1, 4, 5)) + opaque_pixels = np.arange(20, dtype=np.float32).reshape(4, 5) + 100 + opaque = scalar_metadata.payload_with(opaque_pixels) + producer = _runtime() + pattern = compile_function_pattern(_identity, {}, {}) + plan = replace(producer.request.execution_plan, output_dir=tmp_path, step_scope_id="mixed-producer", compiled_function_pattern=pattern) + producer.request.execution_plan = plan + files = _MemoryFiles() + context = ProcessingContext(axis_id="A01", filemanager=files) + context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + producer.request.context = context + producer.pattern_repr = "mixed" + loaded = PatternGroupData(matching_files=[source, source], main_data_stack=planes) + + # A prior homogeneous producer caches the same physical paths. Mixed replacement + # must invalidate that cache rather than leave a false joint runtime domain. + old_bundle = ImageOutputBundle((planes, declared_metadata.payload_with(opaque_pixels[None])), contexts) + old_records = producer._save_outputs(producer._validate_and_unstack(old_bundle, loaded), [source, source]) + assert context.runtime_image_stack_cache.get(tuple(r.output_path for r in old_records), memory_type=MEMORY_TYPE_NUMPY) is not None + bundle = ImageOutputBundle((planes, opaque), contexts) + output = producer._validate_and_unstack(bundle, loaded) + records = producer._save_outputs(output, [source, source]) + manifest = step_output_manifest(context) + manifest.record_outputs(plan, records) + assert tuple(r.output_path for r in records) == tuple(r.output_path for r in old_records) + assert [r.main_flow_plane_axis for r in records] == [RuntimePlaneAxis.RUNTIME_SLICE, None] + assert context.runtime_image_stack_cache.get(tuple(r.output_path for r in records), memory_type=MEMORY_TYPE_NUMPY) is None + monkeypatch.setattr(SourceBindingRuntimeContextRequest, "from_context", classmethod(lambda cls, **kwargs: SimpleNamespace(runtime_context=SourceBindingRuntimeContext.empty))) + + def consumer_for(compiled): + consumer_plan = replace(plan, step_index=1, step_scope_id="mixed-consumer", input_dir=tmp_path, + compiled_function_pattern=compiled, + main_input_dependency=StepInputDependency.step_output(source_step_index=0, source_step_scope_id=plan.step_scope_id)) + consumer = PatternGroupRuntime(SimpleNamespace(context=context, execution_plan=consumer_plan, + compiled_group=compiled.default_group, source_binding_plan=CompiledSourceBindingPlan.empty(), + component_key=None, pattern_group_info="A01_s001_w1_z{iii}_t001.tif")) + monkeypatch.setattr(consumer, "source_workspace_projection_authority", lambda: SimpleNamespace(projection_if_available=lambda: None)) + return consumer + + with pytest.raises(ValueError, match="different declared image axes"): + consumer_for(pattern)._load_input_stack() + for name, expected, axis in (("Planes", image_payload_data(planes), RuntimePlaneAxis.RUNTIME_SLICE), ("Volume", opaque_pixels, None)): + selected = consumer_for(_selected_pattern(name)) + actual = selected._load_input_stack().main_data_stack + np.testing.assert_array_equal(image_payload_data(actual), expected) + assert image_payload_metadata(actual).plane_axis is axis + if not cache_hit: + context.runtime_image_stack_cache.clear() + else: + selected_path = next(r.output_path for r in records if r.producer_identity.output_key == name) + context.runtime_image_stack_cache.store((selected_path,), memory_type=MEMORY_TYPE_NUMPY, stack=actual) + repeated = selected._load_input_stack().main_data_stack + np.testing.assert_array_equal(image_payload_data(repeated), expected) + assert image_payload_metadata(repeated).plane_axis is axis + + +def test_multiple_unprojected_named_binding_domains_fail_existing_bundle_grammar(): + contexts = tuple(AlignedImageSliceContext.main_flow(name) for name in ("First", "Second")) + payloads = tuple(ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.SOURCE_BINDING).payload_with( + np.ones((1, 4, 5), dtype=np.float32)) for _ in contexts) + with pytest.raises(ValueError, match="plane axis to be projected"): + _runtime()._validate_and_unstack(ImageOutputBundle(payloads, contexts), PatternGroupData(matching_files=[], main_data_stack=payloads[0])) + + +def test_opaque_named_bundle_uses_existing_same_slice_mask_composition(): + from openhcs.core.aligned_image_payload import ImagePayloadBundleContext + contexts = tuple(AlignedImageSliceContext.main_flow(name) for name in ("First", "Second")) + first_mask = np.ones((4, 5), dtype=bool) + second_mask = first_mask.copy() + second_mask[0, 0] = False + payloads = tuple(ImagePayloadMetadata(source_image_names=(name,)).payload_with( + np.ones((4, 5), dtype=np.float32) * i, mask) + for i, (name, mask) in enumerate(zip(("First", "Second"), (first_mask, second_mask), strict=True))) + output = _runtime()._validate_and_unstack(ImageOutputBundle(payloads, contexts), PatternGroupData(matching_files=[], main_data_stack=payloads[0])) + expected = ImagePayloadBundleContext.from_payloads(payloads).compose() + assert image_payload_metadata(output.stack_payload).plane_axis is RuntimePlaneAxis.SOURCE_BINDING + np.testing.assert_array_equal(image_payload_data(output.stack_payload), image_payload_data(expected)) + np.testing.assert_array_equal(image_payload_mask(output.stack_payload), image_payload_mask(expected)) + + +def test_input_bundle_composition_retains_lazy_plan_device_resolution(tmp_path, monkeypatch): + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + records = tuple(ProducedOutputSemantics.from_output( + plan, tmp_path / f"A01_s001_w1_z001_t001_{name}.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, extension=".tif", source="test"), + main_flow_plane_axis=None, + ) for name in ("First", "Second")) + payloads = tuple(ImagePayloadMetadata(source_image_names=(name,)).payload_with( + np.ones((4, 5), dtype=np.float32) * i) for i, name in enumerate(("First", "Second"))) + + def unexpected_device_resolution(self, memory_type): + raise AssertionError("BUNDLE composition must retain its own memory allocation authority") + + monkeypatch.setattr(CompiledStepPlan, "device_id_for", unexpected_device_resolution) + output = PatternGroupData.from_loaded_images( + [r.output_path for r in records], payloads, + producer_records=records, source_binding_context=SourceBindingRuntimeContext.empty(), + execution_plan=plan, source_projection=None, workspace_source_lookups=(), + ) + assert image_payload_metadata(output.main_data_stack).plane_axis is RuntimePlaneAxis.SOURCE_BINDING + np.testing.assert_array_equal(image_payload_data(output.main_data_stack), np.stack([image_payload_data(p) for p in payloads])) diff --git a/tests/unit/test_persisted_image_metadata.py b/tests/unit/test_persisted_image_metadata.py index 7df244dfc0..6dd325823b 100644 --- a/tests/unit/test_persisted_image_metadata.py +++ b/tests/unit/test_persisted_image_metadata.py @@ -33,7 +33,13 @@ SourceProjectionMetadataSerializer, ) from openhcs.core.source_spatial_domain import SourceSpatialDomain -from openhcs.core.virtual_workspace_metadata import FIELDS, AtomicMetadataWriter +from openhcs.core.virtual_workspace_metadata import ( + FIELDS, + AtomicMetadataWriter, + MetadataWriteError, + VirtualWorkspaceSourceProjectionEntries, +) +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser from openhcs.serialization.json import to_jsonable @@ -189,6 +195,122 @@ def merge(well): assert len(subdir[FIELDS.SOURCE_METADATA]) == 4 +def calibrated_projection(well, path, pixel_size): + spacing = SourceVoxelSpacing((pixel_size, pixel_size)) + source_metadata = {} + spacing.merge_into(source_metadata, path=path) + return SourcePlaneProjection( + OpenHCSPlaneAddress.from_values(well, 1, 1, 1, 1), + SourcePixelRef("disk", path), + source_metadata=source_metadata, + image_metadata=metadata_fixture().replace_fields(source_voxel_spacing=spacing), + ) + + +def publish_projection_inventory(writer, path, projection_metadata, saved_paths): + writer.publish_source_projection_metadata( + path, + "images", + projection_metadata, + serializer=SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()), + saved_image_paths=saved_paths, + microscope_handler_name="VirtualWorkspaceMicroscopeHandler", + source_filename_parser_name="SourceSchemaFilenameParser", + component_labels={}, + backend="disk", + is_main=True, + results_dir=None, + ) + + +@pytest.mark.parametrize( + "merge_method", ("merge_subdirectory_metadata", "merge_source_projection_metadata") +) +def test_atomic_projection_merge_refreshes_geometry_from_current_document( + tmp_path, merge_method +): + path = tmp_path / "openhcs_metadata.json" + writer = AtomicMetadataWriter() + serializer = SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()) + first = calibrated_projection("A01", "images/first.tif", 0.5) + second = calibrated_projection("A02", "images/second.tif", 2.0) + first_fields = serializer.projection_fields(((first, "images/first.tif"),)) + writer.replace_subdirectory_metadata(path, "images", first_fields) + second_fields = serializer.projection_fields(((second, "images/second.tif"),)) + if merge_method == "merge_subdirectory_metadata": + # This API replaces the projection field with the authoritative new list. + writer.merge_subdirectory_metadata(path, {"images": second_fields}) + expected_pixel_size = 2.0 + else: + writer.merge_source_projection_metadata(path, "images", second_fields) + expected_pixel_size = 1.0 # Conflicting calibration has no physical scalar. + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert subdir[FIELDS.PIXEL_SIZE] == expected_pixel_size + + +def test_final_publication_prunes_geometry_and_preserves_other_directories(tmp_path): + path = tmp_path / "openhcs_metadata.json" + writer = AtomicMetadataWriter() + serializer = SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()) + retained = calibrated_projection("A01", "images/retained.tif", 0.5) + deleted = calibrated_projection("A02", "images/deleted.tif", 2.0) + elsewhere = calibrated_projection("A03", "other/retained.tif", 0.5) + projection_paths = ( + (retained, "images/retained.tif"), + (deleted, "images/deleted.tif"), + (elsewhere, "other/retained.tif"), + ) + writer.merge_source_projection_metadata( + path, "images", serializer.projection_fields(projection_paths) + ) + before = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert before[FIELDS.PIXEL_SIZE] == 1.0 + publish_projection_inventory(writer, path, None, ("images/retained.tif",)) + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert subdir[FIELDS.PIXEL_SIZE] == 0.5 + assert subdir[FIELDS.IMAGE_FILES] == ["images/retained.tif"] + entries = VirtualWorkspaceSourceProjectionEntries.from_subdirectory(subdir).entries + assert set(entries) == {"images/retained.tif", "other/retained.tif"} + assert entries["images/retained.tif"].image_metadata == retained.image_metadata + assert entries["other/retained.tif"].image_metadata == elsewhere.image_metadata + # A failed final inventory cannot publish fields or prune existing records. + before_bytes = path.read_bytes() + with pytest.raises(MetadataWriteError, match="lack typed produced addresses"): + publish_projection_inventory(writer, path, None, ("images/unowned.tif",)) + assert path.read_bytes() == before_bytes + + +def test_concurrent_step_publication_retains_records_outside_partial_inventory( + tmp_path, +): + path = tmp_path / "openhcs_metadata.json" + writer = AtomicMetadataWriter() + serializer = SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()) + paths = tuple(f"images/{well}.tif" for well in ("A01", "A02", "A03", "A04")) + + def publish(item): + index, image_path = item + projection = calibrated_projection(f"A0{index + 1}", image_path, 0.5) + publish_projection_inventory( + writer, + path, + serializer.projection_fields(((projection, image_path),)), + (image_path,), + ) + + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(publish, enumerate(paths))) + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert set(subdir[FIELDS.WORKSPACE_MAPPING]) == set(paths) + assert len(subdir[FIELDS.IMAGE_FILES]) == 1 # Most recent partial step inventory. + assert subdir[FIELDS.PIXEL_SIZE] == 0.5 + publish_projection_inventory(writer, path, None, paths) + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert subdir[FIELDS.IMAGE_FILES] == list(paths) + assert len(subdir[FIELDS.SOURCE_PROJECTION]) == len(paths) + assert subdir["wells"] == {f"A0{index + 1}": None for index in range(4)} + + def test_metadata_codec_rejects_unknown_declarations(): with pytest.raises(ValueError, match="undeclared"): ImagePayloadMetadata.from_mapping({"imagined_tile_axis": 9}) diff --git a/tests/unit/test_persisted_roi_only_publication.py b/tests/unit/test_persisted_roi_only_publication.py new file mode 100644 index 0000000000..7338a41787 --- /dev/null +++ b/tests/unit/test_persisted_roi_only_publication.py @@ -0,0 +1,313 @@ +"""Saved-output publication only: synthetic bytes, no ROI or pixel decoding.""" + +from dataclasses import replace +from types import SimpleNamespace + +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +import json + +import pytest + +from openhcs.agent.dto.plate import PlateFileQueryRequest +from openhcs.agent.path_policy import AgentPathPolicy +from openhcs.agent.services.plate_inspection_service import ( + PlateInspectionFileManagerFactory, + PlateInspectionService, +) +from openhcs.constants.constants import Backend +from openhcs.core.artifacts import ArtifactOutputPlan, MetadataArtifactType +from openhcs.core.compiled_step_plan import ( + CompiledStepPlan, + RuntimeArtifactMaterializationPlan, +) +from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.runtime_artifact_values import RuntimeValue +from openhcs.core.runtime_stores import RuntimeArtifactLocation, StoredRuntimeValue +from openhcs.core.steps.function_artifact_materialization import ( + MaterializedRuntimeArtifact, + RuntimeArtifactMaterialization, +) +from openhcs.core.steps.function_outputs import ( + OpenHCSMetadataWriter, + ProducedImageMetadataCapability, + RuntimeArtifactMetadataTarget, +) +from openhcs.core.virtual_workspace_metadata import METADATA_CONFIG, MetadataWriteError +from openhcs.processing.materialization.core import ( + MaterializationSpec, + _WRITERS_BY_OPTIONS, + prepare_materialization, +) +from openhcs.processing.materialization.options import FileBundleOptions + + +@pytest.fixture +def publication_context(tmp_path): + filemanager = FileManager({Backend.DISK.value: DiskStorageBackend()}) + context = ProcessingContext(filemanager=filemanager) + context.metadata_cache = {} + context.microscope_handler = SimpleNamespace( + parser=SourceSchemaFilenameParser(), microscope_type="openhcsdata" + ) + context.publication_audit = [] + return context + + +class PublicationAuditCapability: + """Independent audit capability extends, rather than replaces, publication.""" + + def write(self, context, *, produced_plan=None): + context.publication_audit.append(("before", self.sub_dir)) + super().write(context, produced_plan=produced_plan) + context.publication_audit.append(("after", self.sub_dir)) + + +def plan_for(plate): + return CompiledStepPlan( + step_index=0, + step_name="synthetic saved result", + step_type="FunctionStep", + axis_id="A01", + write_backend=Backend.MEMORY.value, + create_openhcs_metadata=True, + output_plate_root=str(plate), + analysis_results_dir=str(plate / "compiled_default"), + runtime_artifact_materialization=RuntimeArtifactMaterializationPlan( + persistent_enabled=True, + persistent_backend=Backend.DISK.value, + ), + ) + + +def save_archive_outcome(context, plate, destination, monkeypatch): + """Use the original declared bundle writer, batch and disk save boundary.""" + spec = MaterializationSpec(FileBundleOptions()) + payload = { + f"{destination}/independent-outline.roi.zip": b"synthetic admission only" + } + base_path = plate / "bundle" + batch = prepare_materialization( + spec, + payload, + str(base_path), + context.filemanager, + (Backend.DISK.value,), + ) + saved = batch.save() + (output,) = saved.outputs_for_backend(Backend.DISK.value) + assert output is batch.outputs[0] + # Source qualification must consume the original successful save, not render + # a mutable logical payload again or use planned names as proof of a save. + payload.clear() + + def forbidden_render(*args, **kwargs): + pytest.fail("Publication rendered the already-saved batch again") + + writer = _WRITERS_BY_OPTIONS[FileBundleOptions] + monkeypatch.setitem( + _WRITERS_BY_OPTIONS, FileBundleOptions, replace(writer, write=forbidden_render) + ) + artifact_plan = ArtifactOutputPlan( + name="synthetic archive bundle", + path="/memory/bundle.pkl", + artifact_type=MetadataArtifactType, + materialization=spec, + ) + record = StoredRuntimeValue( + value=RuntimeValue.from_output_plan( + artifact_plan, + {}, + execution_scope=RuntimeExecutionAxisScope(axis_id="A01"), + ), + location=RuntimeArtifactLocation( + path=artifact_plan.path, backend=Backend.MEMORY.value + ), + ) + materialization = RuntimeArtifactMaterialization( + output_plan=artifact_plan, + spec=spec, + record=record, + data=payload, + base_path=base_path, + source_identity=None, + filename_source_identity=None, + ) + return ( + MaterializedRuntimeArtifact( + outputs_by_backend=saved.outputs_by_backend, + materialization=materialization, + ), + output, + ) + + +class ExistingDiskInspectionFactory(PlateInspectionFileManagerFactory): + """Original public service receives the same real disk manager as saving.""" + + def __init__(self, manager): + self.manager = manager + + def create(self): + return self.manager + + +@pytest.fixture(autouse=True) +def forbid_optional_backend_bootstrap(monkeypatch): + import polystore.base + + def forbidden_bootstrap(*args, **kwargs): + pytest.fail("Source-only qualification must not bootstrap optional backends") + + monkeypatch.setattr(polystore.base, "ensure_storage_registry", forbidden_bootstrap) + + +def result_paths(plate, context): + service = PlateInspectionService( + AgentPathPolicy.with_roots(readable_roots=(plate,), writable_roots=()), + filemanager_factory=ExistingDiskInspectionFactory(context.filemanager), + ) + result = service.query_files( + PlateFileQueryRequest.from_fields( + plate_path=str(plate), + microscope_type="openhcsdata", + kind="result", + include_previews=False, + ) + ) + assert result.errors == () + assert all(record.preview is None for record in result.records) + return tuple(record.full_path for record in result.records) + + +@pytest.mark.parametrize("destination", ("roi_only", "independent/nested/roi_only")) +def test_saved_roi_only_batch_publishes_and_reconciles_without_rendering( + tmp_path, + publication_context, + monkeypatch, + destination, +): + plate = tmp_path / "plate" + outcome, output = save_archive_outcome( + publication_context, plate, destination, monkeypatch + ) + plan = plan_for(plate) + targets = OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, + plan, + artifact_materializations=(outcome,), + ) + assert tuple(target.output_dir for target in targets) == (plate / destination,) + OpenHCSMetadataWriter.write( + publication_context, plan, artifact_materializations=(outcome,) + ) + assert result_paths(plate, publication_context) == (output.path,) + published = json.loads(METADATA_CONFIG.metadata_path(plate).read_text()) + assert published["subdirectories"][destination]["image_files"] == [] + assert published["subdirectories"][destination]["source_projection"] == [] + # Final reconciliation has no runtime outcome/value to reconstruct. Its + # destination comes from the same durable declaration the reader admitted. + publication_context.step_plans = {0: plan} + owner = RuntimeArtifactMetadataTarget.from_plan(plan) + reconciled = tuple( + target.output_dir + for target in owner.reconciliation_targets(publication_context) + ) + assert plate / destination in reconciled + OpenHCSMetadataWriter.finalize_completed_plate({"A01": publication_context}) + assert result_paths(plate, publication_context) == (output.path,) + + +def test_shared_writer_accepts_result_only_but_not_unaddressed_images( + tmp_path, + publication_context, +): + plate = tmp_path / "plate" + directory = plate / "standalone_result" + directory.mkdir(parents=True) + roi = directory / "independent-outline.roi.zip" + roi.write_bytes(b"synthetic admission only") + target = RuntimeArtifactMetadataTarget.from_plan(plan_for(plate)).for_directory( + directory + ) + target.write(publication_context) + assert result_paths(plate, publication_context) == (str(roi),) + metadata_path = METADATA_CONFIG.metadata_path(plate) + before = metadata_path.read_bytes() + (directory / "unaddressed-image.tif").write_bytes(b"never decoded") + with pytest.raises( + MetadataWriteError, match="Saved images lack typed produced addresses" + ): + target.write(publication_context) + assert metadata_path.read_bytes() == before + + +def test_new_result_declaration_composes_cooperative_hooks_without_consumer_edits( + tmp_path, + publication_context, + monkeypatch, +): + plate = tmp_path / "plate" + destination = "new/declaration_only" + outcome, output = save_archive_outcome( + publication_context, plate, destination, monkeypatch + ) + registry = OpenHCSMetadataWriter.OutputTarget.__registry__ + original_declarations = dict(registry) + try: + + class IndependentSavedResultMetadataTarget( + PublicationAuditCapability, + ProducedImageMetadataCapability, + RuntimeArtifactMetadataTarget, + ): + declaration_key = None + + @classmethod + def from_plan(cls, plan): + return cls( + output_dir=plate / destination, + backend=Backend.DISK.value, + plate_root=str(plate), + sub_dir=destination, + results_dir=str(plate / destination), + ) + + plan = replace( + plan_for(plate), + runtime_artifact_materialization=RuntimeArtifactMaterializationPlan(), + ) + (target,) = OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, + plan, + artifact_materializations=(outcome,), + ) + target.write(publication_context, produced_plan=plan) + assert publication_context.publication_audit == [ + ("before", destination), + ("after", destination), + ] + assert result_paths(plate, publication_context) == (output.path,) + finally: + registry.clear() + registry.update(original_declarations) + + +def test_files_without_successful_batch_outcomes_do_not_create_step_targets( + tmp_path, + publication_context, +): + plate = tmp_path / "plate" + directory = plate / "unclaimed" + directory.mkdir(parents=True) + (directory / "not_a_saved_outcome.roi.zip").write_bytes(b"synthetic admission only") + assert ( + OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, + plan_for(plate), + artifact_materializations=(), + ) + == () + ) diff --git a/tests/unit/test_preparation_admission.py b/tests/unit/test_preparation_admission.py index 3ed203e53d..99c99a3a55 100644 --- a/tests/unit/test_preparation_admission.py +++ b/tests/unit/test_preparation_admission.py @@ -189,8 +189,10 @@ def prepare_selected(): ) dynamic_invocation = CompiledFunctionInvocation( key=FunctionInvocationKey("dynamic", "default", 1), - contract=CallableContract.from_callable(dynamic), + contract=CallableContract.from_prepared_callable(dynamic), ) + assert events == ["selected", "unselected", "dynamic"] + assert dynamic_invocation.contract.metadata.canonical_signature is not None group = CompiledFunctionGroup("default", (invocation, dynamic_invocation)) pattern = CompiledFunctionPattern(groups=(group,), is_grouped=False) context = SimpleNamespace(step_plans={0: SimpleNamespace(step_index=0, compiled_function_pattern=pattern)}) diff --git a/tests/unit/test_rescale_stream_plane_declaration.py b/tests/unit/test_rescale_stream_plane_declaration.py index 50e2cc6c1b..981d2d3c79 100644 --- a/tests/unit/test_rescale_stream_plane_declaration.py +++ b/tests/unit/test_rescale_stream_plane_declaration.py @@ -16,7 +16,7 @@ from openhcs.core.component_group_scope import RuntimeExecutionAxisScope from openhcs.core.function_patterns import InvocationArtifactInputEdgePlan from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest -from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis, RuntimePlaneAxisValueProjection from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.core.config import StepSourceBindingsConfig from openhcs.core.source_bindings import NamedSourceBinding @@ -162,7 +162,16 @@ def test_declared_source_plane_has_area_not_volume(retained_site): plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, ) payload = metadata.payload_with(pixels, None) - objects = SourceImageObjectLabelBuildRequest(image=payload, labels=labels).payload() + objects = SourceImageObjectLabelBuildRequest( + image=payload, + labels=labels, + plane_projection=( + RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ) + if retained_site else None + ), + ).payload() _, rows = measure_object_size_shape.__wrapped__( payload, objects, calculate_advanced=False, calculate_zernikes=False, ) diff --git a/tests/unit/test_runtime_adapter_source_artifacts.py b/tests/unit/test_runtime_adapter_source_artifacts.py index 28595b345a..dd4e4357fe 100644 --- a/tests/unit/test_runtime_adapter_source_artifacts.py +++ b/tests/unit/test_runtime_adapter_source_artifacts.py @@ -2,6 +2,7 @@ from types import SimpleNamespace import numpy as np +import pytest from polystore.base import ensure_storage_registry, storage_registry from polystore.filemanager import FileManager from scipy.io import savemat @@ -12,7 +13,9 @@ from openhcs.core.component_group_scope import RuntimeExecutionAxisScope from openhcs.core.runtime_adapters import RuntimeAdapterRequest from openhcs.core.runtime_image_loading import ImagePayloadSourceMetadataContext -from openhcs.core.runtime_image_values import image_payload_data +from openhcs.core.runtime_image_values import image_payload_data, image_payload_metadata +from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache +from openhcs.core.source_binding_selection import SourcePatternResolutionContext from openhcs.core.runtime_object_labels import ObjectLabelSet from openhcs.core.source_binding_workspace import SourceBindingWorkspaceProjector from openhcs.core.source_bindings import ( @@ -54,6 +57,7 @@ def _filemanager() -> FileManager: def test_source_artifact_inputs_share_workspace_vfs_and_contract_resolution( tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: source_root = tmp_path / "source" workspace_root = tmp_path / "workspace" @@ -141,6 +145,7 @@ def binding( filemanager=filemanager, microscope_handler=microscope_handler, runtime_source_workspace_projection_cache=projection_cache, + runtime_source_binding_context_cache=RuntimeSourceBindingContextCache(), source_image_set_identity_policy=SourceImageSetIdentityPolicy( frozenset((AllComponents.CHANNEL,)) ), @@ -215,12 +220,11 @@ def request() -> RuntimeAdapterRequest: ) illumination_request = request() + illumination_payload = illumination_request.source_artifact_payload( + illumination_binding.input_spec().ref() + ) np.testing.assert_array_equal( - image_payload_data( - illumination_request.source_artifact_payload( - illumination_binding.input_spec().ref() - ) - ), + image_payload_data(illumination_payload), illumination[np.newaxis, ...], ) @@ -245,3 +249,37 @@ def request() -> RuntimeAdapterRequest: ) assert isinstance(label_set, ObjectLabelSet) np.testing.assert_array_equal(label_set.labels, labels[np.newaxis, ...]) + + # Exercise the actual source adapter and VFS under the independent live + # selector route. Payload identity belongs to the unchanged typed source + # projection, not the selector's resolved metadata record representation. + with monkeypatch.context() as live_selector: + live_selector.setattr( + RuntimeSourceBindingContextCache, + "source_pattern_context", + lambda self, **kwargs: SourcePatternResolutionContext.from_projection( + **kwargs + ), + ) + live_illumination = request().source_artifact_payload( + illumination_binding.input_spec().ref() + ) + live_labels = request().source_artifact_payload(labels_spec.ref()) + for produced, live in ( + (illumination_payload, live_illumination), + (labels_payload, live_labels), + ): + np.testing.assert_array_equal( + image_payload_data(produced), image_payload_data(live) + ) + produced_metadata = image_payload_metadata(produced) + live_metadata = image_payload_metadata(live) + assert produced_metadata == live_metadata + assert ( + produced_metadata.source_provenance.source_identity.identity + == live_metadata.source_provenance.source_identity.identity + ) + assert ( + produced_metadata.source_provenance.represented_source_identities + == live_metadata.source_provenance.represented_source_identities + ) diff --git a/tests/unit/test_runtime_execution_validation.py b/tests/unit/test_runtime_execution_validation.py index 077d4a5e41..9906fed3bb 100644 --- a/tests/unit/test_runtime_execution_validation.py +++ b/tests/unit/test_runtime_execution_validation.py @@ -33,7 +33,11 @@ from openhcs.core.measurement_row_materialization import ( MeasurementSparseColumnarRows, ) -from openhcs.core.orchestrator.execution_result import ExecutionResult +from openhcs.core.orchestrator.execution_result import ( + ExecutionResult, + RuntimeContextObservation, + RuntimeExecutionObservation, +) from openhcs.core.pipeline.function_contracts import execution_scope from openhcs.core.runtime_artifact_values import ( ArtifactKey, @@ -120,7 +124,8 @@ def _streaming_context( def test_runtime_execution_validation_detects_missing_artifact_kind() -> None: observation = RuntimeArtifactExecutionObservation.from_contexts( - {"A01": ProcessingContext(axis_id="A01")} + {"A01": ProcessingContext(axis_id="A01")}, + runtime_observations=(), ) failures = runtime_artifact_execution_failures( @@ -186,7 +191,9 @@ def test_runtime_execution_observation_reads_context_stores() -> None: backend="memory", ) - observation = RuntimeArtifactExecutionObservation.from_contexts({"A01": context}) + observation = RuntimeArtifactExecutionObservation.from_contexts( + {"A01": context}, runtime_observations=() + ) assert observation.record_counts_by_axis["A01"][MeasurementsArtifactType] == 1 @@ -204,7 +211,9 @@ def test_runtime_execution_observation_ignores_uncontracted_files( context = ProcessingContext(axis_id="A01") - observation = RuntimeArtifactExecutionObservation.from_contexts({"A01": context}) + observation = RuntimeArtifactExecutionObservation.from_contexts( + {"A01": context}, runtime_observations=() + ) assert observation.exports.table_outputs == () assert observation.exports.image_outputs == () @@ -239,16 +248,23 @@ def test_zmq_observation_exports_exact_compiler_owned_artifacts( }, axis_id="A01", ) - monkeypatch.setattr( - "openhcs.core.steps.function_artifact_materialization." - "runtime_export_artifact_output_paths", - lambda _plan, _context: (contracted_output,), + runtime_observations = ( + RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + context_key="A01", + records=(), + runtime_export_paths=(contracted_output,), + ), + ) + ), ) export = ZMQRuntimeExecutionObservationExport.from_execution( compiled_contexts={"A01": context}, execution_results={"A01": ExecutionResult.success("A01")}, output_roots=(tmp_path,), + runtime_observations=runtime_observations, ) assert export.expectation.artifact_kinds == frozenset((MeasurementsArtifactType,)) @@ -258,7 +274,9 @@ def test_zmq_observation_exports_exact_compiler_owned_artifacts( assert export.exports.output_files == (contracted_output,) assert unrelated_output not in export.exports.output_files assert ( - RuntimeExportObservation.from_execution_contexts({"A01": context}).output_files + RuntimeExportObservation.from_runtime_observations( + runtime_observations + ).output_files == export.exports.output_files ) @@ -514,22 +532,22 @@ def test_runtime_execution_observation_reads_plate_export_from_exact_owner( ) for axis_id, owns_output in (("A01", True), ("A02", False)) } - observed_axes: list[str] = [] - - def output_paths(plan, _context): - observed_axes.append(plan.axis_id) - return (contracted_output,) - - monkeypatch.setattr( - "openhcs.core.steps.function_artifact_materialization." - "runtime_export_artifact_output_paths", - output_paths, + runtime_observations = ( + RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + context_key="A01", + records=(), + runtime_export_paths=(contracted_output,), + ), + ) + ), + ) + observation = RuntimeArtifactExecutionObservation.from_contexts( + contexts, runtime_observations=runtime_observations ) - - observation = RuntimeArtifactExecutionObservation.from_contexts(contexts) expectation = RuntimeArtifactExecutionExpectation.from_compiled_contexts(contexts) - assert observed_axes == ["A01"] assert observation.exports.table_outputs == (contracted_output,) assert expectation.artifact_kinds == frozenset((MeasurementsArtifactType,)) assert tuple( @@ -578,6 +596,7 @@ def test_zmq_observation_compresses_and_preserves_exact_runtime_records( compiled_contexts={"A01": context}, execution_results={"A01": ExecutionResult.success("A01")}, output_roots=(tmp_path,), + runtime_observations=(), ) path = tmp_path / "observation.pkl" diff --git a/tests/unit/test_runtime_output_matching.py b/tests/unit/test_runtime_output_matching.py index 74eb8a2e32..968ecb23ec 100644 --- a/tests/unit/test_runtime_output_matching.py +++ b/tests/unit/test_runtime_output_matching.py @@ -21,6 +21,12 @@ FunctionStepExecutionScope, ) from openhcs.core.runtime_output_matching import RuntimeReturnedOutputMatcher +from openhcs.core.runtime_image_values import ImagePayloadMetadata, image_payload_data +from openhcs.core.runtime_plane_projection import ( + RuntimePlaneAxis, + RuntimePlaneAxisValueProjection, +) +from openhcs.core.runtime_slice_projection import RuntimeSliceProjectionDeclarationError from openhcs.core.function_patterns import ( CompiledFunctionInvocation, FunctionInvocationKey, @@ -59,6 +65,58 @@ def test_runtime_output_matcher_maps_canonical_and_trailing_slots() -> None: } +def test_matcher_contextualizes_declared_axis_before_resolving_complete_abi() -> None: + first = ArtifactSpec.output("First", ImageArtifactType) + second = ArtifactSpec.output("Second", ImageArtifactType) + measurements = ArtifactSpec.output("Measurements", MeasurementsArtifactType) + data = np.arange(24, dtype=np.float32).reshape((2, 3, 4)) + payload = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE + ).payload_with(data, None) + trailing = object() + contract = _contract(first, second, measurements) + matcher = RuntimeReturnedOutputMatcher(contract, (payload, trailing)) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=2, + ) + + returned = matcher.contextualize_canonical_output(plane_projection=projection) + resolved = RuntimeReturnedOutputMatcher(contract, returned).resolve() + + assert returned[1] is trailing + assert resolved[measurements.ref()] is trailing + for index, spec in enumerate((first, second)): + selected = image_payload_data(resolved[spec.ref()]) + np.testing.assert_array_equal(selected, data[index]) + assert np.shares_memory(selected, data) + assert ( + RuntimeReturnedOutputMatcher(contract, returned).contextualize_canonical_output() + is returned + ) + + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="without a compiled"): + matcher.contextualize_canonical_output() + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="already selected"): + matcher.contextualize_canonical_output( + plane_projection=projection.selected_plane(0) + ) + with pytest.raises(ValueError, match="projection declares 3 value"): + matcher.contextualize_canonical_output( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=3, + ) + ) + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="plane axis"): + matcher.contextualize_canonical_output( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.SOURCE_BINDING, + axis_size=2, + ) + ) + + def test_runtime_output_matcher_uses_exact_multi_canonical_contexts() -> None: first = ArtifactSpec.output("First", ImageArtifactType) second = ArtifactSpec.output("Second", ImageArtifactType) diff --git a/tests/unit/test_runtime_slice_projection.py b/tests/unit/test_runtime_slice_projection.py index 78aec1511f..a3cc39fdf8 100644 --- a/tests/unit/test_runtime_slice_projection.py +++ b/tests/unit/test_runtime_slice_projection.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +import inspect from typing import Any, get_args, get_origin import numpy as np @@ -85,6 +86,8 @@ def _annotation_leaf_types(annotation: object) -> tuple[type, ...]: def test_cellprofiler_runtime_parameter_types_declare_slice_projection() -> None: missing: list[str] = [] for parameter_type in _subclasses(KeywordRuntimeParameter): + if inspect.isabstract(parameter_type): + continue if not parameter_type.__module__.startswith( ("openhcs.processing.backends.cellprofiler", "openhcs.interop.cellprofiler") ): diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py new file mode 100644 index 0000000000..5d3660d75c --- /dev/null +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -0,0 +1,332 @@ +"""Resolution lifetimes across generic source selectors and artifact adapters.""" + +import pickle +from types import MappingProxyType + +import pytest +from polystore.virtual_workspace import SourcePixelRef + +from openhcs.constants.constants import AllComponents +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache +from openhcs.core.source_binding_selection import ( + DeclaredSourceMetadataRecord, + SourceBindingMatchedImageSet, + SourcePatternResolutionContext, +) +from openhcs.core.source_bindings import ( + MetadataExtractionRule, + MetadataSource, + SourceBindingRuntimeContext, +) +from openhcs.core.source_matching import SourceImageSetIdentityPolicy +from openhcs.core.source_metadata import ( + ORIGINAL_SOURCE_METADATA_FIELD, + ResolvedSourceMetadataRecord, +) +from openhcs.core.source_projection import OpenHCSPlaneAddress, SourcePlaneProjection +from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection +from openhcs.microscopes.microscope_interfaces import FilenameParseResult +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser + +PATH = "A01_s001_w1_z001_t001.tif" + + +class CountingParser(SourceSchemaFilenameParser): + def __init__(self): + super().__init__() + self.calls = [] + + def parse_filename(self, path): + self.calls.append(path) + result = super().parse_filename(path) + if result is None or self.pattern_format is None: + return result + return FilenameParseResult( + tuple( + ( + component, + ( + int(self.pattern_format) + if component is AllComponents.SITE + else value + ), + ) + for component, value in result.declared_values() + ), + extension=result.extension, + ) + + +def projection(metadata): + return VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path={PATH: SourcePixelRef("disk", "/source/image.tif")}, + source_metadata_by_path={PATH: metadata}, + ) + + +def snapshot(cache, source_projection, parser, rules=()): + return cache.source_pattern_context( + parser=parser, projection=source_projection, metadata_rules=rules + ) + + +def test_declared_record_resolves_live_parser_and_nested_metadata(): + nested = {"Plate": "before"} + record = DeclaredSourceMetadataRecord.from_mapping( + {ORIGINAL_SOURCE_METADATA_FIELD: nested, "site": 8} + ) + assert isinstance(record, DeclaredSourceMetadataRecord) + parser = CountingParser() + first = record.resolve(PATH, parser, ()) + nested["Plate"] = "after" + second = record.resolve(PATH, parser, ()) + assert first["site"] == second["site"] == 8 + assert first[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "before" + assert second[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "after" + assert parser.calls == [PATH, PATH] + + +def test_runtime_snapshot_owns_nested_metadata_and_unknown_fallback(): + nested = {"Plate": "before"} + source_projection = projection({ORIGINAL_SOURCE_METADATA_FIELD: nested}) + parser = CountingParser() + cache = RuntimeSourceBindingContextCache() + context = snapshot(cache, source_projection, parser) + calls = tuple(parser.calls) + nested["Plate"] = "after" + for _ in range(3): + metadata = context.metadata_for_path(PATH) + assert isinstance(metadata, ResolvedSourceMetadataRecord) + assert metadata[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "before" + assert tuple(parser.calls) == calls + with pytest.raises(TypeError): + metadata[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] = "invalid" + unknown = "A02_s001_w1_z001_t001.tif" + assert context.metadata_for_path(unknown)["well"] == "A02" + assert parser.calls[-1] == unknown + assert snapshot(cache, source_projection, parser) is context + entry = next(iter(cache.source_resolution_snapshots.values())) + assert entry.projection is source_projection + assert entry.context.parser is parser + + +def test_direct_projection_context_remains_live(): + nested = {"Plate": "before"} + parser = CountingParser() + context = SourcePatternResolutionContext.from_projection( + parser=parser, + projection=projection({ORIGINAL_SOURCE_METADATA_FIELD: nested}), + ) + assert ( + context.metadata_for_path(PATH)[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] + == "before" + ) + nested["Plate"] = "after" + assert ( + context.metadata_for_path(PATH)[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] + == "after" + ) + assert parser.calls == [PATH, PATH] + + +def test_snapshot_replacement_tracks_projection_parser_semantics_and_rules(): + cache = RuntimeSourceBindingContextCache() + parser = CountingParser() + source_projection = projection({}) + original = snapshot(cache, source_projection, parser) + parser.pattern_format = "9" + changed_parser = snapshot(cache, source_projection, parser) + assert changed_parser is not original + assert original.metadata_for_path(PATH)["site"] == 1 + assert changed_parser.metadata_for_path(PATH)["site"] == 9 + replaced_projection = snapshot(cache, projection({"site": 7}), parser) + assert replaced_projection.metadata_for_path(PATH)["site"] == 7 + replaced_parser = snapshot(cache, source_projection, CountingParser()) + assert replaced_parser is not original + assert replaced_parser.metadata_for_path(PATH)["site"] == 1 + rules = ( + MetadataExtractionRule( + source=MetadataSource.FILE_NAME, pattern=r"(?PA01)" + ), + ) + changed_rules = snapshot(cache, source_projection, parser, rules) + assert changed_rules is not changed_parser + assert ( + changed_rules.metadata_for_path(PATH)[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] + == "A01" + ) + + +def test_matched_set_preserves_resolved_record_behavior_and_precedence(): + parser = CountingParser() + context = snapshot( + RuntimeSourceBindingContextCache(), projection({"site": None}), parser + ) + matched = SourceBindingMatchedImageSet.from_plan( + bindings=(), + match_plan=None, + source_context=context, + identity_policy=SourceImageSetIdentityPolicy(), + ) + calls = tuple(parser.calls) + assert ( + matched.source_metadata_by_path[PATH] is context.source_metadata_by_path[PATH] + ) + assert matched.metadata_for_path(PATH)["site"] == 1 + assert tuple(parser.calls) == calls + + +def test_virtual_positions_remain_distinct_with_shared_physical_address(): + other = "A01_s002_w1_z001_t001.tif" + source_projection = VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path={ + PATH: SourcePixelRef("disk", "/source/shared.tif"), + other: SourcePixelRef("disk", "/source/shared.tif"), + }, + source_metadata_by_path=MappingProxyType( + {PATH: {"site": 1}, other: {"site": 2}} + ), + ) + context = snapshot( + RuntimeSourceBindingContextCache(), source_projection, CountingParser() + ) + assert context.virtual_paths_for_source("/source/shared.tif") == (PATH, other) + assert context.metadata_for_path(PATH)["site"] == 1 + assert context.metadata_for_path(other)["site"] == 2 + + +def test_known_empty_record_remains_none_without_repeated_parsing(): + parser = CountingParser() + source_projection = VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path={"opaque": SourcePixelRef("disk", "opaque")}, + source_metadata_by_path={}, + ) + context = snapshot(RuntimeSourceBindingContextCache(), source_projection, parser) + calls = tuple(parser.calls) + assert context.metadata_for_path("opaque") is None + assert context.metadata_for_path("opaque") is None + assert tuple(parser.calls) == calls + + +def test_snapshot_rejects_nested_values_outside_existing_metadata_grammar(): + with pytest.raises(TypeError, match="Source metadata scalar values"): + snapshot( + RuntimeSourceBindingContextCache(), + projection({"nested": {"unsupported": {"deep": 1}}}), + CountingParser(), + ) + + +def test_declared_and_resolved_records_preserve_family_value_identity(): + declared = DeclaredSourceMetadataRecord.from_mapping({"site": 1, "well": "A01"}) + resolved = ResolvedSourceMetadataRecord.from_mapping({"site": 1, "well": "A01"}) + assert declared == resolved + assert hash(declared) == hash(resolved) + assert hash(declared) == hash((declared.fields,)) + assert declared != DeclaredSourceMetadataRecord.from_mapping( + {"site": 2, "well": "A01"} + ) + nested = {ORIGINAL_SOURCE_METADATA_FIELD: {"Plate": "A"}} + with pytest.raises(TypeError): + hash(DeclaredSourceMetadataRecord.from_mapping(nested)) + with pytest.raises(TypeError): + hash(ResolvedSourceMetadataRecord.from_mapping(nested)) + + +def test_snapshot_owns_position_and_projection_map_views(): + original_ref = SourcePixelRef("disk", "/source/original.tif") + refs = {PATH: original_ref} + positions = { + PATH: SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values("A01", 1, 1, 1, 1), + ref=original_ref, + source_alias="original", + ) + } + original_projection = positions[PATH] + source_projection = VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path=refs, + source_metadata_by_path={PATH: {}}, + source_projections_by_virtual_path=positions, + ) + context = snapshot( + RuntimeSourceBindingContextCache(), source_projection, CountingParser() + ) + refs[PATH] = SourcePixelRef("disk", "/source/replaced.tif") + positions.clear() + assert context.source_paths_by_virtual_path[PATH] == "/source/original.tif" + assert context.source_projections_by_virtual_path[PATH] is original_projection + with pytest.raises(TypeError): + context.source_projections_by_virtual_path[PATH] = original_projection + + +def test_resolved_direct_constructor_owns_deep_immutable_invariant(): + nested = {"Plate": "before"} + resolved = ResolvedSourceMetadataRecord(((ORIGINAL_SOURCE_METADATA_FIELD, nested),)) + nested["Plate"] = "after" + assert resolved[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "before" + with pytest.raises(TypeError): + resolved[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] = "invalid" + with pytest.raises(TypeError, match="Source metadata scalar values"): + ResolvedSourceMetadataRecord((("nested", {"unsupported": {"deep": 1}}),)) + + +@pytest.mark.parametrize("context_owned", (False, True)) +def test_warmed_cache_transport_reconstructs_all_derived_defaults(context_owned): + processing_context = ProcessingContext(axis_id="A01") + cache = ( + processing_context.runtime_source_binding_context_cache + if context_owned + else RuntimeSourceBindingContextCache() + ) + parser = CountingParser() + source_projection = projection({ORIGINAL_SOURCE_METADATA_FIELD: {"Plate": "A"}}) + context = snapshot(cache, source_projection, parser) + cache.normalized_source_metadata(source_projection.source_metadata_by_path) + cache.runtime_context_by_request_identity[(1, (), "disk", None)] = ( + SourceBindingRuntimeContext() + ) + assert cache.source_resolution_snapshots + assert cache.source_metadata_by_mapping_identity + restored_owner = pickle.loads( + pickle.dumps(processing_context if context_owned else cache) + ) + restored = ( + restored_owner.runtime_source_binding_context_cache + if context_owned + else restored_owner + ) + if context_owned: + assert restored_owner.axis_id == "A01" + assert restored == RuntimeSourceBindingContextCache() + rebuilt = snapshot(restored, source_projection, parser) + assert rebuilt is not context + assert rebuilt.metadata_for_path(PATH) == context.metadata_for_path(PATH) + + +def test_durable_workspace_mapping_still_runs_selector_parser_and_rule_fallbacks(): + from openhcs.core.source_metadata import DurableSourceMetadata, SourceMetadataRecord + + metadata = DurableSourceMetadata.from_mapping({"site": 8, "literal": "kept"}) + parser = CountingParser() + context = SourcePatternResolutionContext.from_sources( + parser=parser, + source_paths_by_virtual_path={PATH: "/source/image.tif"}, + source_metadata_by_path={PATH: metadata}, + metadata_rules=( + MetadataExtractionRule( + source=MetadataSource.FILE_NAME, pattern=r"(?PA01)" + ), + ), + ) + assert not isinstance(metadata, SourceMetadataRecord) + assert isinstance( + context.source_metadata_by_path[PATH], DeclaredSourceMetadataRecord + ) + resolved = context.metadata_for_path(PATH) + assert resolved["site"] == 8 + assert resolved["well"] == "A01" + assert resolved["literal"] == "kept" + assert resolved[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "A01" + assert parser.calls == [PATH] diff --git a/tests/unit/test_runtime_stack_cache_lifetime.py b/tests/unit/test_runtime_stack_cache_lifetime.py index 78470ea371..f735926289 100644 --- a/tests/unit/test_runtime_stack_cache_lifetime.py +++ b/tests/unit/test_runtime_stack_cache_lifetime.py @@ -38,7 +38,7 @@ def test_release_drops_all_cached_views_and_allows_reuse(context_owned: bool) -> assert all(cache.get(entry, memory_type="numpy") is None for entry in paths) replacement = np.zeros((1, 8, 8), dtype=np.uint16) cache.store(paths[0], memory_type="numpy", stack=replacement) - assert cache.get(paths[0], memory_type="numpy").stack is replacement + assert cache.get(paths[0], memory_type="numpy") is replacement def test_context_image_cache_release_preserves_runtime_artifacts_and_observations() -> ( diff --git a/tests/unit/test_saved_source_label_plane_domain.py b/tests/unit/test_saved_source_label_plane_domain.py index 2383b24259..a8c9f6c7ef 100644 --- a/tests/unit/test_saved_source_label_plane_domain.py +++ b/tests/unit/test_saved_source_label_plane_domain.py @@ -41,6 +41,34 @@ def _saved_label_input(source): ) +@pytest.mark.parametrize("axis", tuple(RuntimePlaneAxis)) +@pytest.mark.parametrize("plane_count", (1, 2)) +def test_explicit_object_projection_preserves_axis_without_acquisition_provenance( + axis, plane_count, +): + labels = np.ones((plane_count, 6, 7), dtype=np.int32) + source = ImagePayloadMetadata(plane_axis=axis).payload_with(labels) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=axis, axis_size=plane_count, + ) + payload = SourceImageObjectLabelBuildRequest( + image=source, labels=labels, plane_projection=projection, + ).payload() + + assert payload.plane_axis is axis + assert payload.domain.declared_object_id_domains == ((1,),) * plane_count + assert payload.source_provenance == source.metadata.source_provenance + np.testing.assert_array_equal(payload.labels, labels) + other_axis = next(member for member in RuntimePlaneAxis if member is not axis) + with pytest.raises(ValueError, match="conflicts with the source-image axis"): + SourceImageObjectLabelBuildRequest( + image=source, labels=labels, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=other_axis, axis_size=plane_count, + ), + ).payload() + + @pytest.mark.parametrize("axis", tuple(RuntimePlaneAxis)) def test_original_artifact_admission_and_full_stack_shape_keep_2d_planes(axis): labels = np.zeros((2, 8, 9), dtype=np.int32) diff --git a/tests/unit/test_source_binding_workspace.py b/tests/unit/test_source_binding_workspace.py index 32436c2527..94c7c0c6c5 100644 --- a/tests/unit/test_source_binding_workspace.py +++ b/tests/unit/test_source_binding_workspace.py @@ -12,7 +12,7 @@ from openhcs.core.source_binding_workspace import SourceBindingWorkspaceProjector from openhcs.core.runtime_tabular_values import FieldSpec from openhcs.core.source_metadata import ( - SourceMetadataRoleView, + SourceMetadataFields, SourceVoxelSpacing, SourceVoxelSpacingUnit, SOURCE_VOXEL_SPACING_FIELD, @@ -330,9 +330,7 @@ def test_source_binding_workspace_projector_assigns_selector_channels(tmp_path): } assert all("image_type" not in source for source in serialized_sources) filter_paths_by_alias = { - str(source["source_alias"]): SourceMetadataRoleView( - source - ).source_filter_paths() + str(source["source_alias"]): SourceMetadataFields.source_filter_paths(source) for source in serialized_sources } assert filter_paths_by_alias == { @@ -386,7 +384,7 @@ def test_source_binding_workspace_projects_semantic_identity_after_raw_selection assert projection.address.value_for(AllComponents.CHANNEL) == "MCP_DNA" original_metadata = dict( - SourceMetadataRoleView(projection.source_metadata).original_items() + SourceMetadataFields.original_items(projection.source_metadata) ) assert original_metadata[AllComponents.CHANNEL.value] == "1" @@ -609,7 +607,7 @@ def test_group_address_preserves_distinct_source_well_as_literal_metadata(tmp_pa source_metadata = next(iter(metadata[FIELDS.SOURCE_METADATA].values())) assert source_metadata["well"] == "Sequence1" - assert dict(SourceMetadataRoleView(source_metadata).original_items()) == { + assert dict(SourceMetadataFields.original_items(source_metadata)) == { "Well": "A01", "FrameNumber": "0000", "Run": "Sequence1", @@ -788,7 +786,7 @@ def test_order_source_sets_join_imported_metadata_across_aliases(tmp_path): assert {projection.source_alias for projection in projections} == {"DNA", "Actin"} for projection in projections: original = dict( - SourceMetadataRoleView(projection.source_metadata).original_items() + SourceMetadataFields.original_items(projection.source_metadata) ) assert original["Compound"] == "DMSO" assert original["Dose"] == "0" @@ -912,7 +910,7 @@ def test_metadata_source_sets_propagate_imported_metadata(tmp_path): assert len(projections) == 2 assert all( - dict(SourceMetadataRoleView(item.source_metadata).original_items())["Compound"] + dict(SourceMetadataFields.original_items(item.source_metadata))["Compound"] == "DrugA" for item in projections ) @@ -958,7 +956,7 @@ def test_imported_metadata_duplicate_join_uses_first_matching_row(tmp_path): assert metadata["Site"] == "1" assert metadata["Compound"] == "First" - original = dict(SourceMetadataRoleView(metadata).original_items()) + original = dict(SourceMetadataFields.original_items(metadata)) assert original["Site"] == "1" assert original["Plate"] == "20585" assert original["Well"] == "A01" @@ -1006,7 +1004,7 @@ def test_imported_metadata_later_stage_overrides_extracted_field(tmp_path): assert metadata["Plate"] == "plate_1" assert metadata["Dose"] == "0" assert projection.address.value_for(AllComponents.WELL) == "A01" - original = dict(SourceMetadataRoleView(metadata).original_items()) + original = dict(SourceMetadataFields.original_items(metadata)) assert original["Plate"] == "plate_1" assert original["Well"] == "A01" assert original["ChannelNumber"] == "2" @@ -1058,7 +1056,7 @@ def test_imported_metadata_coerces_join_and_payload_values_through_declared_type filemanager=_filemanager(), ) projection = projection_set.projections[0] - original = dict(SourceMetadataRoleView(projection.source_metadata).original_items()) + original = dict(SourceMetadataFields.original_items(projection.source_metadata)) assert original["Site"] == 5 assert original["Dose"] == 0.25 @@ -1072,7 +1070,7 @@ def test_imported_metadata_coerces_join_and_payload_values_through_declared_type pixel_size=1.0, ) serialized = next(iter(metadata[FIELDS.SOURCE_METADATA].values())) - serialized_original = dict(SourceMetadataRoleView(serialized).original_items()) + serialized_original = dict(SourceMetadataFields.original_items(serialized)) assert serialized_original["Dose"] == 0.25 assert serialized_original["Frame"] == 7 @@ -1161,7 +1159,7 @@ def test_imported_metadata_skips_source_sets_with_partial_join_identity(tmp_path ) assert "Compound" not in dict( - SourceMetadataRoleView(projection.source_metadata).original_items() + SourceMetadataFields.original_items(projection.source_metadata) ) @@ -1285,7 +1283,7 @@ def test_source_bindings_reinitialization_resolves_current_imported_metadata(tmp ) ) assert ( - dict(SourceMetadataRoleView(first_metadata).original_items())["Compound"] + dict(SourceMetadataFields.original_items(first_metadata))["Compound"] == "First" ) @@ -1300,7 +1298,7 @@ def test_source_bindings_reinitialization_resolves_current_imported_metadata(tmp assert metadata_path.read_bytes() != first_payload assert ( - dict(SourceMetadataRoleView(second_metadata).original_items())["Compound"] + dict(SourceMetadataFields.original_items(second_metadata))["Compound"] == "Second" ) diff --git a/tests/unit/test_source_image_provenance_hotpath.py b/tests/unit/test_source_image_provenance_hotpath.py index bf3e167d50..552cdadf8e 100644 --- a/tests/unit/test_source_image_provenance_hotpath.py +++ b/tests/unit/test_source_image_provenance_hotpath.py @@ -14,7 +14,7 @@ def test_empty_explicit_provenance_reuses_authoritative_value() -> None: ) fields = SourceImageProvenanceFields(source_provenance=retained) - fields.absorb_explicit_source_provenance(SourceImageProvenance()) + fields.absorb_explicit_source_provenance((None, None, None, ())) assert fields.source_provenance is retained @@ -30,7 +30,9 @@ def test_populated_explicit_provenance_preserves_merge_precedence() -> None: ) fields = SourceImageProvenanceFields(source_provenance=fallback) - fields.absorb_explicit_source_provenance(explicit) + fields.absorb_explicit_source_provenance( + (None, {"channel": "2"}, None, ("Explicit",)) + ) assert fields.source_provenance == SourceImageProvenance( source_path="/input/A01_s001_w1.tif", diff --git a/tests/unit/test_source_metadata_live_mutation.py b/tests/unit/test_source_metadata_live_mutation.py index 19cf5086cd..935fc63373 100644 --- a/tests/unit/test_source_metadata_live_mutation.py +++ b/tests/unit/test_source_metadata_live_mutation.py @@ -3,7 +3,7 @@ import pytest from openhcs.constants.constants import AllComponents -from openhcs.core.source_binding_selection import SourceMetadataRecord +from openhcs.core.source_binding_selection import DeclaredSourceMetadataRecord from openhcs.core.source_matching import ( ORIGINAL_SOURCE_METADATA_FIELD, semantic_source_metadata_value, @@ -31,7 +31,7 @@ def test_metadata_queries_observe_scalar_mutation(readonly_outer): @pytest.mark.parametrize( - "view", (dict, MappingProxyType, SourceMetadataRecord.from_mapping) + "view", (dict, MappingProxyType, DeclaredSourceMetadataRecord.from_mapping) ) def test_literal_queries_observe_nested_original_mutation(view): original = {"Well": "LiteralA"} diff --git a/tests/unit/test_source_metadata_owner.py b/tests/unit/test_source_metadata_owner.py new file mode 100644 index 0000000000..f204c09c61 --- /dev/null +++ b/tests/unit/test_source_metadata_owner.py @@ -0,0 +1,676 @@ +"""Owned metadata lifetime and its production provenance consumers.""" + +import pickle +from collections.abc import Mapping, Sequence + +import cloudpickle +from types import MappingProxyType + +import numpy as np +import pytest +from polystore.virtual_workspace import SourcePixelRef + +from openhcs.constants.constants import AllComponents +from openhcs.core.artifacts import ImageArtifactType +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_metadata, +) +from openhcs.core.source_binding_selection import DeclaredSourceMetadataRecord +from openhcs.core.source_bindings import ( + SOURCE_BINDING_ALIAS_METADATA_FIELD, + SourceBindingRuntimeContext, +) +from openhcs.core.source_image_provenance import ( + SourceImageIdentity, + SourceImageProvenance, + SourceImageProvenancePlanes, + SourceImageProvenance, + SourceImageProvenancePlaneRecord, + SourcePlaneIndexedMetadata, + source_component_metadata_consensus, +) +from openhcs.core.source_matching import ( + source_component_metadata_items, + source_component_metadata_value, + source_component_metadata_values, + source_metadata_value, + with_source_component_metadata, +) +from openhcs.core.source_metadata import ( + ORIGINAL_SOURCE_METADATA_FIELD, + DurableSourceMetadata, + ResolvedSourceMetadataRecord, + SourceMetadataRecord, + SourceMetadataFields, + source_metadata_scalar, +) +from openhcs.core.source_projection import OpenHCSPlaneAddress, SourcePlaneProjection +from openhcs.core.steps.function_output_identity import FunctionOutputIdentity +from openhcs.core.source_workspace_projection import ( + VirtualWorkspaceImagePayloadProjection, + VirtualWorkspaceSourceProjection, +) +from openhcs.core.virtual_workspace_metadata import ( + VirtualWorkspaceSourceMetadataEntries, +) +from openhcs.serialization.json import to_jsonable + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +def test_owned_snapshot_detaches_nested_fields_and_preserves_record_contract(owner): + nested = {"Well": "literal"} + record = owner.from_mapping({"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: nested}) + nested["Well"] = "changed" + assert source_metadata_value(record, "Well") == "literal" + with pytest.raises(TypeError): + record[ORIGINAL_SOURCE_METADATA_FIELD]["Well"] = "changed" + with pytest.raises(TypeError): + hash(record) + declared = DeclaredSourceMetadataRecord.from_mapping( + {"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: {"Well": "literal"}} + ) + reversed_record = owner.from_mapping( + {ORIGINAL_SOURCE_METADATA_FIELD: {"Well": "literal"}, "well": "A01"} + ) + if isinstance(record, SourceMetadataRecord): + assert record == declared + assert record != reversed_record + else: + assert record != declared and declared != record + assert record == reversed_record + assert record == dict(declared) + + +def test_owned_lookup_keeps_first_duplicate_field_and_full_ordered_record(): + record = ResolvedSourceMetadataRecord((("well", "A01"), ("well", "A02"))) + assert record["well"] == "A01" + assert tuple(record.items()) == (("well", "A01"), ("well", "A01")) + assert record.fields == (("well", "A01"), ("well", "A02")) + assert dict(record) == {"well": "A01"} + assert record != ResolvedSourceMetadataRecord((("well", "A01"),)) + + +def test_runtime_birth_canonicalizes_but_durable_and_derived_values_keep_literal_spelling( + tmp_path, +): + spelling = str(tmp_path / "unused" / ".." / "image.tif") + durable = DurableSourceMetadata.from_mapping({"path": spelling}) + runtime = ResolvedSourceMetadataRecord.from_mapping(durable) + assert durable["path"] == spelling + assert runtime["path"] == str(tmp_path / "image.tif") + derived = SourceMetadataFields.with_fields(runtime, {"new_path": spelling}) + assert isinstance(derived, DurableSourceMetadata) + assert derived["path"] == str(tmp_path / "image.tif") + assert derived["new_path"] == spelling + projection = SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values( + well="A01", site="1", channel="1", z_index="1", timepoint="1" + ), + ref=SourcePixelRef("disk", "plane.tif"), + source_metadata=derived, + ) + assert projection.source_metadata["new_path"] == str(tmp_path / "image.tif") + + +def test_owned_scalar_identity_reuses_literal_metadata_but_not_mutable_carriers(): + fields = {"well": "A01", "channel": "1", "extension": ".tif"} + owner = DurableSourceMetadata.from_mapping(fields) + original = ImagePayloadMetadata( + source_provenance=SourceImageProvenance( + source_path="/input/source.tif", + source_component_metadata=owner, + source_image_names=("DNA",), + ) + ) + first = original.replace_fields() + second = original.with_source_context_from(original) + assert first.source_provenance is not original.source_provenance + assert ( + first.source_provenance.source_identity + is not original.source_provenance.source_identity + ) + assert first.source_component_metadata is owner + assert second.source_component_metadata is owner + assert ( + first.source_provenance.equality_identity + == original.source_provenance.equality_identity + ) + assert to_jsonable(first) == to_jsonable(original) + first.source_provenance.source_identity.path = "/changed.tif" + assert original.source_path == "/input/source.tif" + assert second.source_path == "/input/source.tif" + + +def test_workspace_alias_projection_and_artifact_naming_preserve_owned_metadata(): + source = VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields( + {"well": "A01", "channel": "1", SOURCE_BINDING_ALIAS_METADATA_FIELD: "DNA"} + ) + payload = ImagePayloadMetadata().payload_with(np.ones((2, 2), dtype=np.float32)) + projected = VirtualWorkspaceImagePayloadProjection( + source_metadata=source, source_alias="DNA" + ).apply(payload) + metadata = image_payload_metadata(projected) + assert isinstance(metadata.source_component_metadata, DurableSourceMetadata) + assert SOURCE_BINDING_ALIAS_METADATA_FIELD not in metadata.source_component_metadata + named = ImageArtifactType.normalize_runtime_payload("DerivedDNA", projected) + named_metadata = image_payload_metadata(named) + assert ( + named_metadata.source_component_metadata is metadata.source_component_metadata + ) + assert named_metadata.source_image_names == ("DerivedDNA",) + assert metadata.source_image_names == ("DNA",) + assert ( + named_metadata.source_provenance.source_image_provenance_planes.represented_source_image_names + == ("DNA",) + ) + + +def test_owned_updates_replace_aliases_and_keep_original_literals(): + owner = DurableSourceMetadata.from_mapping( + { + "channel": "1", + "ChannelNumber": "2", + ORIGINAL_SOURCE_METADATA_FIELD: {"ChannelNumber": "literal"}, + } + ) + updated = with_source_component_metadata(owner, AllComponents.CHANNEL, "3") + assert isinstance(updated, DurableSourceMetadata) + assert "ChannelNumber" not in updated + assert source_metadata_value(updated, "ChannelNumber") == "literal" + assert source_component_metadata_values(updated, AllComponents.CHANNEL) == ("3",) + assert owner["channel"] == "1" + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +def test_custom_scalar_remains_live_for_queries_and_new_fingerprints(owner): + class MutableInteger(int): + def __str__(self): + return self.tag + + def __repr__(self): + return self.tag + + value = MutableInteger(1) + value.tag = "before" + record = owner.from_mapping({"channel": value}) + first = SourceImageIdentity(component_metadata=record) + assert source_component_metadata_value(record, AllComponents.CHANNEL) == "before" + value.tag = "after" + assert source_component_metadata_value(record, AllComponents.CHANNEL) == "after" + assert SourceImageIdentity(component_metadata=record).identity != first.identity + assert first.identity[1] == (("channel", "before"),) + + +def test_lazy_role_errors_are_unchanged_by_owned_construction(): + record = DurableSourceMetadata.from_mapping( + { + "channel": "1", + "wellrow": "A", + "wellcolumn": "bad", + ORIGINAL_SOURCE_METADATA_FIELD: 7, + } + ) + assert source_component_metadata_value(record, AllComponents.CHANNEL) == "1" + with pytest.raises(ValueError): + source_component_metadata_value(record, AllComponents.WELL) + with pytest.raises(RuntimeError, match="must be a mapping"): + source_metadata_value(record, "channel") + with pytest.raises(RuntimeError, match="must be a mapping"): + source_component_metadata_values(record, AllComponents.CHANNEL) + + +@pytest.mark.parametrize("invalid", ([1], {"deep": {"unsupported": 1}}, object())) +def test_durable_metadata_keeps_existing_invalid_value_domain(invalid): + with pytest.raises(RuntimeError): + VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields( + {"field": invalid} + ) + + +def test_source_identity_mapping_equality_keeps_original_class_and_current_field_boundaries(): + owner = DurableSourceMetadata.from_mapping({"flag": True}) + identity = SourceImageIdentity("source.tif", owner) + raw = SourceImageIdentity("source.tif", MappingProxyType({"flag": True})) + assert identity == raw and raw == identity + assert identity != SourceImageIdentity("source.tif", {"flag": 1}) + assert SourceImageIdentity(component_metadata=None) != SourceImageIdentity( + component_metadata={} + ) + with pytest.raises(TypeError): + hash(identity) + + class Child(SourceImageIdentity): + pass + + assert identity.__eq__(Child("source.tif", owner)) is NotImplemented + + class ReportedIdentity: + @property + def __class__(self): + return SourceImageIdentity + + def __getattribute__(self, name): + if name in ("path", "component_metadata", "_identity"): + reads.append(name) + return getattr(raw, name) + return object.__getattribute__(self, name) + + reads = [] + assert identity.__eq__(ReportedIdentity()) is True + assert reads == ["path", "component_metadata", "_identity"] + + nested = {"literal": "before"} + left = SourceImageIdentity( + component_metadata={ORIGINAL_SOURCE_METADATA_FIELD: nested} + ) + right = SourceImageIdentity( + component_metadata={ORIGINAL_SOURCE_METADATA_FIELD: {"literal": "before"}} + ) + captured = left.identity + assert left == right + nested["literal"] = "after" + assert left != right and left.identity == captured + + +def test_nested_mapping_order_does_not_change_provenance_identity_or_wire_order(): + raw = {"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: {"site": "001", "channel": "1"}} + reordered = {ORIGINAL_SOURCE_METADATA_FIELD: {"channel": "1", "site": "001"}, "well": "A01"} + owned = DurableSourceMetadata.from_mapping(reordered) + before = to_jsonable(owned) + original = SourceImageProvenance(source_path="source.tif", source_component_metadata=raw) + for metadata in (reordered, owned): + candidate = SourceImageProvenance(source_path="source.tif", source_component_metadata=metadata) + assert candidate == original + assert candidate.equality_identity == original.equality_identity + assert SourceMetadataFields.provenance_identity_items(raw) == SourceMetadataFields.provenance_identity_items(owned) + assert to_jsonable(owned) == before + assert tuple(reordered) == (ORIGINAL_SOURCE_METADATA_FIELD, "well") + assert tuple(reordered[ORIGINAL_SOURCE_METADATA_FIELD]) == ("channel", "site") + reordered[ORIGINAL_SOURCE_METADATA_FIELD]["site"] = "002" + changed = SourceImageProvenance(source_path="source.tif", source_component_metadata=reordered) + assert changed.equality_identity != original.equality_identity + + +def test_canonical_metadata_identity_preserves_ordered_provenance_planes(): + paths = ("a.tif", "b.tif") + metadata = ({"site": "1"}, {"site": "2"}) + forward = SourceImageProvenancePlanes.from_components(paths=paths, component_metadata=metadata) + reverse = SourceImageProvenancePlanes.from_components( + paths=tuple(reversed(paths)), component_metadata=tuple(reversed(metadata)), + ) + assert SourceImageProvenance(source_image_provenance_planes=forward).equality_identity != SourceImageProvenance( + source_image_provenance_planes=reverse, + ).equality_identity + + +@pytest.mark.parametrize("serializer", (pickle, cloudpickle)) +def test_identity_transport_preserves_birth_fingerprint_after_current_metadata_changes(serializer): + from openhcs.core.orchestrator.execution_result import RuntimeExecutionTransportSerialization + + RuntimeExecutionTransportSerialization.register() + nested = {"site": "001", "channel": "1"} + identity = SourceImageIdentity("original.tif", {ORIGINAL_SOURCE_METADATA_FIELD: nested}) + birth = identity.identity + nested["site"] = "002" + identity.path = "current.tif" + restored = serializer.loads(serializer.dumps(identity)) + assert restored.path == "current.tif" + assert restored.component_metadata[ORIGINAL_SOURCE_METADATA_FIELD]["site"] == "002" + assert restored.identity == birth + assert SourceImageIdentity(restored.path, restored.component_metadata).identity != birth + + +def test_raw_composition_snapshots_once_before_consensus_reads(): + class ObservableMapping(Mapping): + def __init__(self, values): + self.values = values + self.key_reads = self.value_reads = 0 + + def __iter__(self): + self.key_reads += 1 + return iter(self.values) + + def __len__(self): + return len(self.values) + + def __getitem__(self, key): + self.value_reads += 1 + return self.values[key] + + raw = ObservableMapping({"well": "A01"}) + consensus = source_component_metadata_consensus((raw, raw)) + assert dict(consensus) == {"well": "A01"} + assert raw.key_reads == 2 and raw.value_reads == 2 + snapshot = SourceMetadataFields.composition_snapshot(raw) + raw.values["well"] = "A02" + assert snapshot["well"] == "A01" + assert source_component_metadata_consensus((raw, raw))["well"] == "A02" + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +@pytest.mark.parametrize("serializer", (pickle, cloudpickle)) +def test_owned_transport_stores_only_authoritative_fields_and_rebuilds_local_views( + owner, serializer +): + record = owner.from_mapping( + {"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: {"literal": "value"}} + ) + before = SourceImageIdentity(component_metadata=record).identity + assert source_metadata_value(record, "literal") == "value" + assert source_component_metadata_value(record, AllComponents.WELL) == "A01" + assert record._views + restored = serializer.loads(serializer.dumps(record)) + assert isinstance(restored, owner) + assert restored.fields == record.fields + assert not restored._views + assert restored.metadata_contents is not record.metadata_contents + assert to_jsonable(restored) == to_jsonable(record) + assert SourceImageIdentity(component_metadata=restored).identity == before + with pytest.raises(TypeError): + restored[ORIGINAL_SOURCE_METADATA_FIELD]["literal"] = "changed" + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +def test_transport_does_not_recanonicalize_stored_absolute_values_or_validate_unused_roles( + owner, tmp_path +): + directory = tmp_path / "original" + directory.mkdir() + spelling = str(directory / "image.tif") + record = owner.from_mapping({"path": spelling, ORIGINAL_SOURCE_METADATA_FIELD: 7}) + before = SourceImageIdentity(component_metadata=record).identity + directory.rmdir() + target = tmp_path / "replacement" + target.mkdir() + directory.symlink_to(target, target_is_directory=True) + assert str((directory / "image.tif").resolve()) != spelling + restored = pickle.loads(pickle.dumps(record)) + assert restored["path"] == spelling + assert SourceImageIdentity(component_metadata=restored).identity == before + with pytest.raises(RuntimeError, match="must be a mapping"): + source_metadata_value(restored, "unused") + + +def test_ordered_component_batch_matches_sequential_raw_updates(): + fields = { + "Well": "old", + "Site": "old", + "ChannelNumber": "old", + "ZIndex": "old", + "Timepoint": "old", + ORIGINAL_SOURCE_METADATA_FIELD: {"Well": "literal"}, + } + components = tuple( + (component, str(index)) for index, component in enumerate(AllComponents, 1) + ) + raw = dict(fields) + for component, value in components: + raw = with_source_component_metadata(raw, component, value) + record = DurableSourceMetadata.from_mapping(fields) + updated = SourceMetadataFields.with_fields(record, {}, components=components) + assert isinstance(updated, DurableSourceMetadata) + assert to_jsonable(updated) == to_jsonable(raw) + assert tuple(updated) == tuple(raw) + assert source_metadata_value(updated, "Well") == "literal" + + +@pytest.mark.parametrize("existing_extension", (False, True)) +def test_output_identity_batch_keeps_original_field_order_and_extension_precedence( + existing_extension, +): + fields = {"literal": "kept", "Well": "old"} + if existing_extension: + fields["extension"] = ".old" + identity = FunctionOutputIdentity( + component_values={"well": "A01", "channel": "1"}, + extension=".tif", + source="control", + ) + expected = dict(fields) + expected.update(identity.component_values) + for component, value in source_component_metadata_items(identity.component_values): + expected = with_source_component_metadata(expected, component, value) + expected["extension"] = ".tif" + actual = identity.component_metadata(DurableSourceMetadata.from_mapping(fields)) + assert tuple(actual.items()) == tuple(expected.items()) + assert to_jsonable(actual) == expected + + +def test_duplicate_record_keys_collapse_at_source_identity_mapping_boundary(): + record = ResolvedSourceMetadataRecord((("well", "A01"), ("well", "A02"))) + expected = SourceImageIdentity(component_metadata=dict(record)) + actual = SourceImageIdentity(component_metadata=record) + assert actual == expected + assert actual.identity == expected.identity + assert tuple(actual.component_metadata.items()) == (("well", "A01"),) + assert record.fields == (("well", "A01"), ("well", "A02")) + + +def test_stringified_key_collisions_keep_platform_map_last_value_at_owned_birth(): + fields = {1: "first", "1": "last"} + durable = VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields(fields) + assert tuple(durable.items()) == (("1", "last"),) + projection = SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values( + well="A01", site="1", channel="1", z_index="1", timepoint="1" + ), + ref=SourcePixelRef("disk", "plane.tif"), + source_metadata=fields, + ) + assert tuple(projection.source_metadata.items()) == (("1", "last"),) + declaration = DeclaredSourceMetadataRecord.from_mapping(fields) + assert declaration.fields == (("1", "first"), ("1", "last")) + assert declaration["1"] == "first" + + +@pytest.mark.parametrize( + "decoder", + ( + ResolvedSourceMetadataRecord.from_mapping, + DurableSourceMetadata.from_mapping, + VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields, + ), +) +def test_owned_map_birth_validates_overwritten_stringified_key_values(decoder): + with pytest.raises((TypeError, RuntimeError)): + decoder({1: [], "1": "valid"}) + + +def test_mapping_and_ordered_record_equality_namespaces_stay_separate(): + ordered = ResolvedSourceMetadataRecord.from_mapping({"well": "A01", "site": "1"}) + reordered = ResolvedSourceMetadataRecord.from_mapping({"site": "1", "well": "A01"}) + mapping = DurableSourceMetadata.from_mapping(ordered) + reordered_mapping = DurableSourceMetadata.from_mapping(reordered) + assert ordered != reordered + assert mapping == reordered_mapping + assert mapping == dict(ordered) and dict(ordered) == mapping + for record in ( + ordered, + reordered, + DeclaredSourceMetadataRecord.from_mapping(dict(ordered)), + ): + assert mapping != record and record != mapping + assert hash(ordered) == hash( + DeclaredSourceMetadataRecord.from_mapping(dict(ordered)) + ) + with pytest.raises(TypeError): + hash(mapping) + + +@pytest.mark.parametrize( + "factory", + ( + lambda metadata: VirtualWorkspaceSourceMetadataEntries({"plane.tif": metadata}), + lambda metadata: VirtualWorkspaceImagePayloadProjection( + source_metadata=metadata + ), + lambda metadata: VirtualWorkspaceSourceProjection({}, {"plane.tif": metadata}), + lambda metadata: SourceImageProvenancePlaneRecord(component_metadata=metadata), + lambda metadata: SourcePlaneIndexedMetadata(metadata, 0, 1), + lambda metadata: SourceBindingRuntimeContext( + source_metadata_by_path={"plane.tif": metadata} + ), + lambda metadata: SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values( + well="A01", site="1", channel="1", z_index="1", timepoint="1" + ), + ref=SourcePixelRef("disk", "plane.tif"), + source_metadata=metadata, + ), + ), +) +def test_public_mapping_field_carriers_keep_content_equality(factory): + owner = DurableSourceMetadata.from_mapping({"well": "A01", "site": "1"}) + reordered = DurableSourceMetadata.from_mapping({"site": "1", "well": "A01"}) + raw = MappingProxyType({"well": "A01", "site": "1"}) + assert factory(owner) == factory(reordered) + assert factory(owner) == factory(raw) + assert factory(raw) == factory(owner) + with pytest.raises(TypeError): + hash(factory(owner)) + + +def test_durable_tuple_birth_uses_mapping_last_value_policy(): + metadata = DurableSourceMetadata((("well", "A01"), ("well", "A02"))) + assert metadata.fields == (("well", "A02"),) + assert metadata == {"well": "A02"} + assert ( + SourceImageIdentity(component_metadata=metadata).identity + == SourceImageIdentity(component_metadata={"well": "A02"}).identity + ) + + +def test_scalar_admission_preserves_primitives_subclasses_and_scalar_container_precedence(): + class Integer(int): + pass + + class Real(float): + pass + + class TextMapping(str, Mapping): + pass + + values = ( + None, + False, + True, + 0, + -3, + 1.25, + "literal", + Integer(7), + Real(2.5), + TextMapping("literal"), + ) + for value in values: + assert DurableSourceMetadata.normalized_scalar(value) is value + assert source_metadata_scalar(value) is value + assert isinstance(values[-1], Mapping) + assert isinstance(values[-1], Sequence) + + +def test_durable_scalar_admission_preserves_rejected_container_and_scalar_errors(): + class MappingSequence(Mapping, Sequence): + def __getitem__(self, key): + raise KeyError(key) + + def __iter__(self): + return iter(()) + + def __len__(self): + return 0 + + container_error = ( + "virtual_workspace source metadata supports scalar values and " + "one-level scalar mappings only." + ) + scalar_error = ( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + for value in ( + {}, + [], + (), + range(1), + b"bytes", + bytearray(b"bytes"), + MappingSequence(), + ): + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(value) + assert str(caught.value) == container_error + for value in (set(), frozenset(), 1j, object()): + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(value) + assert str(caught.value) == scalar_error + + +def test_scalar_admission_keeps_none_identity_and_reported_class_read_order(): + class ReportedClass: + def __init__(self, reports): + self.reports = reports + self.reads = [] + + @property + def __class__(self): + reported = self.reports[min(len(self.reads), len(self.reports) - 1)] + self.reads.append(reported) + return reported + + reported_none = ReportedClass((type(None),)) + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(reported_none) + assert str(caught.value) == ( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + assert reported_none.reads == [type(None)] * 6 + + reported_none = ReportedClass((type(None),)) + with pytest.raises(TypeError, match="Source metadata scalar values must be"): + source_metadata_scalar(reported_none) + assert reported_none.reads == [type(None)] * 4 + + reported_integer = ReportedClass((object, int, object)) + assert DurableSourceMetadata.normalized_scalar(reported_integer) is reported_integer + assert reported_integer.reads == [object, int] + + for normalize in ( + source_metadata_scalar, + SourceMetadataFields.normalized_scalar, + ResolvedSourceMetadataRecord.normalized_scalar, + ): + reported_integer = ReportedClass((object, int, object)) + assert normalize(reported_integer) is reported_integer + assert reported_integer.reads == [object, int, object] + + # Scalar admission and string normalization are separate observations. + # A changing reported class must retain the previous normalization order. + reported_string = ReportedClass((str, object)) + assert normalize(reported_string) is reported_string + assert reported_string.reads == [str, object] + + reported_string = ReportedClass((str, object)) + assert DurableSourceMetadata.normalized_scalar(reported_string) is reported_string + assert reported_string.reads == [str] + + changing_class = ReportedClass( + (object, object, object, object, object, Sequence, str) + ) + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(changing_class) + assert str(caught.value) == ( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + assert changing_class.reads == [ + object, + object, + object, + object, + object, + Sequence, + str, + ] diff --git a/tests/unit/test_source_projection.py b/tests/unit/test_source_projection.py index a2a590d50a..5871a9745f 100644 --- a/tests/unit/test_source_projection.py +++ b/tests/unit/test_source_projection.py @@ -11,7 +11,7 @@ from openhcs.core.source_bindings import SourceProjectionRole from openhcs.core.source_metadata import ( ORIGINAL_SOURCE_METADATA_FIELD, - SourceMetadataRoleView, + SourceMetadataFields, ) from openhcs.core.source_projection import ( OpenHCSPlaneAddress, @@ -269,7 +269,7 @@ def test_source_projection_preserves_provenance_owned_component_remaps( assert source_metadata[component.value] == address_value assert ( - dict(SourceMetadataRoleView(source_metadata).original_items())[component.value] + dict(SourceMetadataFields.original_items(source_metadata))[component.value] == source_value ) diff --git a/tests/unit/test_source_provenance_constructor_defaults.py b/tests/unit/test_source_provenance_constructor_defaults.py new file mode 100644 index 0000000000..f25eaa478c --- /dev/null +++ b/tests/unit/test_source_provenance_constructor_defaults.py @@ -0,0 +1,99 @@ +"""All source-metadata carriers decode authored aliases at the same boundary.""" + +from functools import partial + +import numpy as np +import pytest + +from openhcs.core.artifacts import ArtifactSpec, ObjectLabelsArtifactType +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_measurements import ( + MeasurementScope, + MeasurementSubject, + MeasurementTable, +) +from openhcs.core.runtime_object_labels import ( + ObjectLabelPayload, + ObjectLabelSet, + ObjectLabelVariantData, +) +from openhcs.core.runtime_relationships import ( + DirectedObjectRelationshipPayload, + ObjectRelationship, + ObjectRelationshipDeclaration, +) +from openhcs.core.runtime_spatial_graph import SpatialGraph +from openhcs.core.runtime_tabular_values import ColumnarRows +from openhcs.core.source_image_provenance import SourceImageProvenance + + +class EmptyRows(ColumnarRows): + columns = {} + fields = () + + +def _carriers(): + labels = ObjectLabelVariantData(labels=np.zeros((2, 3), dtype=np.int32)) + relationship = ObjectRelationshipDeclaration.parent_child( + source=ArtifactSpec.output("Cells", ObjectLabelsArtifactType).ref(), + target=ArtifactSpec.output("Nuclei", ObjectLabelsArtifactType).ref(), + producer_module_number=1, + ) + return ( + ImagePayloadMetadata, + partial(ObjectLabelPayload, variant_data=labels), + partial(ObjectLabelSet, name="Cells", variant_data=labels), + partial( + MeasurementTable, + name="Intensity", + rows=EmptyRows(), + subject=MeasurementSubject(MeasurementScope.IMAGE, "DNA"), + ), + partial(SpatialGraph, name="Graph", nodes=(), edges=()), + partial( + ObjectRelationship, + name="ParentChild", + declaration=relationship, + payload=DirectedObjectRelationshipPayload(source_ids=(), target_ids=()), + ), + ) + + +@pytest.mark.parametrize("construct", _carriers()) +def test_absent_aliases_do_not_create_empty_provenance(construct, monkeypatch): + supplied = SourceImageProvenance( + source_path="/tmp/source.tif", source_component_metadata={"well": "A01"} + ) + + def unexpected(cls, values): + pytest.fail("Absent source aliases must not allocate a decoded value.") + + monkeypatch.setattr( + SourceImageProvenance, "from_init_values", classmethod(unexpected) + ) + carrier = construct(source_provenance=supplied) + assert carrier.source_path == "/tmp/source.tif" + assert carrier.source_component_metadata["well"] == "A01" + assert carrier.source_provenance is not supplied + carrier.source_provenance.source_identity.path = "/tmp/changed.tif" + assert supplied.source_path == "/tmp/source.tif" + + +@pytest.mark.parametrize("construct", _carriers()) +def test_explicit_empty_metadata_and_aliases_still_decode(construct, monkeypatch): + calls = [] + original = SourceImageProvenance.from_init_values + + def observe(cls, values): + calls.append(values) + return original(values) + + monkeypatch.setattr(SourceImageProvenance, "from_init_values", classmethod(observe)) + empty = construct(source_component_metadata={}) + assert calls == [(None, {}, None, ())] + assert empty.source_component_metadata is not None + assert dict(empty.source_component_metadata) == {} + authored = construct(source_path="/tmp/authored.tif", source_image_names=("DNA",)) + assert authored.source_path == "/tmp/authored.tif" + assert authored.source_image_names == ("DNA",) + assert len(calls) == 2 diff --git a/tests/unit/test_source_provenance_nested_identity.py b/tests/unit/test_source_provenance_nested_identity.py new file mode 100644 index 0000000000..12516f33a9 --- /dev/null +++ b/tests/unit/test_source_provenance_nested_identity.py @@ -0,0 +1,51 @@ +"""Metadata mapping order is not identity; values and plane order are.""" +from collections.abc import Mapping + +from openhcs.core.source_image_provenance import SourceImageIdentity, SourceImageProvenancePlanes +from openhcs.core.source_metadata import OriginalSourceMetadata, SourceFilterPathMetadata + + +def source_records(*, literal_site="001"): + metadata = {"well": "A01", "site": "1", "channel": "1", "z_index": "0"} + OriginalSourceMetadata.from_mapping({"well": "A01", "site": literal_site, "z_index": "000"}).merge_into( + metadata, path="/input/A01_z000.tif", + ) + SourceFilterPathMetadata.from_paths(("A01_z000.tif", "/input/A01_z000.tif")).merge_into( + metadata, path="/input/A01_z000.tif", + ) + return metadata + + +def test_source_identity_nested_mapping_reorder_is_equivalent(): + metadata = source_records() + reordered = { + key: dict(reversed(tuple(value.items()))) if isinstance(value, Mapping) else value + for key, value in reversed(tuple(metadata.items())) + } + before = SourceImageIdentity("/input/A01_z000.tif", metadata) + after = SourceImageIdentity("/input/A01_z000.tif", reordered) + assert dict(before.component_metadata) == metadata + assert dict(after.component_metadata) == reordered + assert before.identity == after.identity + planes = SourceImageProvenancePlanes.from_components + assert planes(paths=(before.path,), component_metadata=(metadata,)) == planes( + paths=(after.path,), component_metadata=(reordered,), + ) + + +def test_source_identity_changed_nested_value_is_distinct(): + original = source_records() + changed = source_records(literal_site="002") + assert SourceImageIdentity("/input/A01_z000.tif", original).identity != SourceImageIdentity( + "/input/A01_z000.tif", changed, + ).identity + + +def test_source_identity_distinct_plane_order_is_not_equivalent(): + first = SourceImageIdentity("/input/A01_z000.tif", source_records()) + second = SourceImageIdentity("/input/A01_z001.tif", {"well": "A01", "z_index": "1"}) + planes = SourceImageProvenancePlanes.from_components + forward = planes(paths=(first.path, second.path), component_metadata=(first.component_metadata, second.component_metadata)) + reverse = planes(paths=(second.path, first.path), component_metadata=(second.component_metadata, first.component_metadata)) + assert forward.identity != reverse.identity + assert forward != reverse diff --git a/tests/unit/test_spatial_graph_materialization.py b/tests/unit/test_spatial_graph_materialization.py index 6dc0145d26..0c0e70ef37 100644 --- a/tests/unit/test_spatial_graph_materialization.py +++ b/tests/unit/test_spatial_graph_materialization.py @@ -17,6 +17,7 @@ ObjectLabelsArtifactType, SpatialGraphArtifactType, ) +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.runtime_spatial_graph import ( @@ -340,7 +341,7 @@ def test_graph_roi_projection_preserves_paths_and_graph_features() -> None: assert roi_output.path == "/tmp/A01_neurite_graph_step3.graph.roi.zip" assert len(roi_output.content) == 3 - first_roi = roi_output.content[0] + first_roi = ROIArchiveSourceMetadata.geometry(roi_output.content)[0] assert isinstance(first_roi.shapes[0], PolylineShape) np.testing.assert_array_equal( first_roi.shapes[0].coordinates, diff --git a/tests/unit/test_streaming_service.py b/tests/unit/test_streaming_service.py index 9625eb1834..5f801971c6 100644 --- a/tests/unit/test_streaming_service.py +++ b/tests/unit/test_streaming_service.py @@ -9,7 +9,7 @@ import pytest from polystore.disk import DiskStorageBackend from polystore.filemanager import FileManager -from polystore.roi import ROI, PointShape +from polystore.roi import ROI, PointShape, load_rois_from_zip from polystore.roi_converters import NapariROIConverter from polystore.streaming.identity import ( FixedStreamProducerIdentityKind, @@ -954,9 +954,10 @@ def test_reopen_native_roi_archives_preserves_per_file_source_and_calibration( stream.source.metadata.metadata_by_path[batch_paths[0]] == expected.source_component_metadata ) - assert data[0][0].metadata == { + assert ROIArchiveSourceMetadata.feature_metadata(data[0][0].metadata) == { "label": expected.source_component_metadata["channel"] } + assert ROIArchiveSourceMetadata.decode(data[0]) == expected assert data[0][0].shapes == [PointShape(32.25, 40.5)] @@ -1030,6 +1031,7 @@ def test_3d_point_archive_reopens_with_native_z_domain_and_features(tmp_path): ), ) config = NapariStreamingConfig(enabled=True) + saved_source = ROIArchiveSourceMetadata.decode(load_rois_from_zip(Path(archive))) result = StreamingService(filemanager, handler, tmp_path).stream_rois( RoiStreamingRequest( viewer=FakeViewer(), @@ -1047,6 +1049,13 @@ def test_3d_point_archive_reopens_with_native_z_domain_and_features(tmp_path): assert paths == [archive] assert data[0][0].metadata["response"] == 4.75 assert data[0][0].metadata["openhcs_fractional_z"] == 2.375 + reopened_source = ROIArchiveSourceMetadata.decode(data[0]) + assert reopened_source is not None + assert reopened_source == saved_source + assert ( + reopened_source.source_image_provenance_planes + == table.source_image_provenance_planes + ) assert stream.source.metadata.metadata_by_path[archive]["z_index"] == 0 assert stream.message_extra[ViewerBatchWireField.COMPONENT_VALUE_DOMAIN.value][ "z_index" diff --git a/tests/unit/test_zmq_execution_server.py b/tests/unit/test_zmq_execution_server.py index 45e8070fa3..85599621bf 100644 --- a/tests/unit/test_zmq_execution_server.py +++ b/tests/unit/test_zmq_execution_server.py @@ -16,6 +16,9 @@ ProcessingConfig, ) from openhcs.core.execution_state import ExecutionOutputPlateSummary +from openhcs.core.orchestrator.compiled_plate_execution import ( + CompiledPlateExecutionResults, +) from openhcs.core.orchestrator.execution_result import ( ExecutionResult, RuntimeContextObservation, @@ -196,20 +199,22 @@ def test_server_exports_outcomes_without_projecting_compiled_values( server._export_runtime_observation( request_context=request_context, compilation=compilation, - execution_results={ - "A01": ExecutionResult.success( - "A01", - runtime_observation=RuntimeExecutionObservation( - contexts=( - RuntimeContextObservation( - "context", - (), - runtime_export_paths=(declared_output,), - ), - ) - ), - ) - }, + execution_results=CompiledPlateExecutionResults( + { + "A01": ExecutionResult.success( + "A01", + runtime_observation=RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + "context", + (), + runtime_export_paths=(declared_output,), + ), + ) + ), + ) + } + ), ) export = ZMQRuntimeExecutionOutcomeExport.read(export_path) diff --git a/tests/unit/test_zmq_runtime_outcome_export.py b/tests/unit/test_zmq_runtime_outcome_export.py index f358c89ca1..041116b678 100644 --- a/tests/unit/test_zmq_runtime_outcome_export.py +++ b/tests/unit/test_zmq_runtime_outcome_export.py @@ -110,6 +110,7 @@ def test_value_export_rejects_an_uncompiled_execution_axis() -> None: compiled_contexts={}, execution_results={"A01": ExecutionResult.success("A01")}, output_roots=(), + runtime_observations=(), ) with pytest.raises(RuntimeError, match="execution outcomes have no compiled axis"): exported.require_valid_observation() @@ -167,6 +168,7 @@ def test_previous_runtime_export_versions_remain_readable(tmp_path: Path) -> Non execution_results={}, output_roots=(), execution_id="new-job", + runtime_observations=(), ), schema_version=6, execution_id=None, @@ -178,6 +180,7 @@ def test_previous_runtime_export_versions_remain_readable(tmp_path: Path) -> Non execution_results={}, output_roots=(), execution_id="other-job", + runtime_observations=(), ).write(observation_path) assert observation_path.read_bytes() == retained_bytes assert (