diff --git a/diagnostics/.gitignore b/diagnostics/.gitignore new file mode 100644 index 0000000000..42bcb8627f --- /dev/null +++ b/diagnostics/.gitignore @@ -0,0 +1,5 @@ +# Raw receipts are preserved in the reviewed tar.gz, byte-for-byte. +*.log +*.xml +*.json +*.stderr diff --git a/diagnostics/ACQUISITION_SHA256SUMS b/diagnostics/ACQUISITION_SHA256SUMS new file mode 100644 index 0000000000..edb5f600a2 --- /dev/null +++ b/diagnostics/ACQUISITION_SHA256SUMS @@ -0,0 +1,15 @@ +3514ad8f2346e5b5e82aaf4410610afc2fc8616eaf4596d5af1eaa387a85efe5 openhcs/core/aligned_image_payload.py +cd6c7553efba0838205f153e96b15570c285894e304827d8c3f3d0c38de664bd openhcs/core/runtime_slice_projection.py +51fa66703e1d789c199ba485f80ec71152bb1f64a3864ed2414d167660ffc2f8 openhcs/interop/cellprofiler/runtime/function_contract_execution.py +939b5cfa8c32eb2db38cee2606ffa755be3eaf96fbe3640674f038ced125f5fc diagnostics/README.rst +2dd04e5005b9cde8b1b9b86f0d17e3b9c45120795e9be0ed9b556787b33ffdf2 diagnostics/run_source_tests.py +9599cb9152d642e834b4056d820856c3dba9593647ea7554c36f0589c72b3798 diagnostics/test_344_engineering_document.py +f398b5db2435208a1317f0e97bf347e7e837e4b64f91ac0887e2e571d6da913c diagnostics/test_344_source_binding_ingress.py +0df290b3cb92862b5aba67a27f45558d0d6f10cd53504f9fffab6f6378a9c296 docs/refactor/344-engineering-acceptance.rst +323ab83b8462a9a8f4e8cc873e446eab1de15a114a3f3a07740ea8b682b7666f docs/refactor/examples/344-aligned-rescale-engineering.py +4679d9be9612f31adb1dd42dad7efae887484466edf7fad6afe7e7937bccf095 diagnostics/engineering-source-bindings-ingress.log +54f76ba7e2240dacadaaf95dedee73620a46dcd5650b79a9bedfec1b6c6a0391 diagnostics/engineering-source-bindings-ingress.xml +0f9b93720ac0fe0c7c3b9ebc78c2dfeb6fa710173b071659e0d1f9072606c459 diagnostics/engineering-source-bindings-identity-controls.log +79b7422ea0f90d73a7de03e2042e20c47af77cdbd9c51f643a76688dc1b57173 diagnostics/engineering-source-bindings-identity-controls.xml +ef0531fa77aa73b5a47cbf40210d516cca94b8dc72330b1de0ab6d9c7ff4397a diagnostics/engineering-acquisition-final-source.log +5381057c14505fa925ff9e070ff3c2ed7ab839d85e7d98b74fb4979503a52994 diagnostics/engineering-acquisition-final-source.xml diff --git a/diagnostics/NOMINAL_REVIEW_SHA256SUMS b/diagnostics/NOMINAL_REVIEW_SHA256SUMS new file mode 100644 index 0000000000..a3e8fb5e5a --- /dev/null +++ b/diagnostics/NOMINAL_REVIEW_SHA256SUMS @@ -0,0 +1,25 @@ +3514ad8f2346e5b5e82aaf4410610afc2fc8616eaf4596d5af1eaa387a85efe5 openhcs/core/aligned_image_payload.py +cd6c7553efba0838205f153e96b15570c285894e304827d8c3f3d0c38de664bd openhcs/core/runtime_slice_projection.py +51fa66703e1d789c199ba485f80ec71152bb1f64a3864ed2414d167660ffc2f8 openhcs/interop/cellprofiler/runtime/function_contract_execution.py +ffc6bfe96a431e72611da4e287d496065af9d4bed71984881a29c3f7a902a1ec tests/unit/test_cellprofiler_full_stack_materialization.py +ae1313ed11505caf35ef1d3e183b811a0d8968a187ff45f73cba1e30a572a63b diagnostics/README.rst +2dd04e5005b9cde8b1b9b86f0d17e3b9c45120795e9be0ed9b556787b33ffdf2 diagnostics/run_source_tests.py +9599cb9152d642e834b4056d820856c3dba9593647ea7554c36f0589c72b3798 diagnostics/test_344_engineering_document.py +b7493d7e12dafec853e02498f6f65979859c5c041010716bd4cf9f4669318fd4 docs/refactor/cp-aligned-full-stack-342.rst +7183385aa0e0a5b79ec9fb343f596c79de29668d608154fdb48c7c0560e26d0b docs/refactor/344-engineering-acceptance.rst +5e0e39229fcfeb34548565dafeb6b04a4dbaa4b40c03603ef65d4f924d6608e2 docs/refactor/examples/344-aligned-rescale-engineering.py +d9a74653eda0f076d505bd42471d09b24c66057ac91d8e5678fe74720bc06526 diagnostics/nominal-review-family-controls.log +6611a228f092fc75374580b61f710f454bf24f6ac61617815d3bc34cc500fe28 diagnostics/nominal-review-family-controls.xml +93bdd1a80cdd48b9ff3ff5c28a0d8ea449f934d9ec212a5c0ddf3f5796225ec0 diagnostics/nominal-review-opaque-controls.log +bdcf8c04e606ffcb6e963283860d24048e6e2ce2a23fd7737a4ebe8058cf7c87 diagnostics/nominal-review-opaque-controls.xml +1c9d31d078d8391c46fd9ff15e2b7a2c52d293be78b22ac4c0a1e28382f9bb21 diagnostics/a14-consumer-controls.log +e79acaefe8b1e66a5b0f95b20c21e5c8040116d7939e178713582396962da39d diagnostics/a14-consumer-controls.xml +c9ac3b4f586accb4d7993b7b20c70f9035a53d195195b7532f61941091609ca9 diagnostics/a14-r0-openhcs.log +96331d5fe50a4bd165a433b503caedbef0c2b4a2cf10689aeb5314f343c7e3ad diagnostics/a14-r0-scripts.log +c1d63a1befd427c68ecb3f5ea30f7e60ce72a361dbb979e36aef74cb46f74307 diagnostics/a14-r0-benchmark.log +5938b383ffbd7bd3899cf642f956e0c12019b9549b4f0f68acaa675beb41ee8e diagnostics/engineering-document-source.log +b9213321e2975c2ec76550c6cfd02592a4b63ad63592061e13021468efe7a1c0 diagnostics/engineering-document-source.xml +0621f1642170818f6c73edf6477a9052f71a1d61163296d2dbb7c3612796892c diagnostics/engineering-document-source-corrected.log +e88365a2b02946daebd1952f360047f6a5de4e76aaa0adaff3ab1d98262b9947 diagnostics/engineering-document-source-corrected.xml +3f823fd0246416a730e68aed4eb63ef1cd4113d2d1d3319b1db74091a7f9e444 diagnostics/engineering-document-separate-planes.log +2a0fd5014902bd62a8ba5638a0de37111d185db93ae27b40d315d6516e66a3ed diagnostics/engineering-document-separate-planes.xml diff --git a/diagnostics/README.rst b/diagnostics/README.rst new file mode 100644 index 0000000000..e019344787 --- /dev/null +++ b/diagnostics/README.rst @@ -0,0 +1,156 @@ +Runtime repair #342 source receipts +================================== + +Latest engineering authoring disposition +--------------------------------------- + +The public example is corrected to the parent's admitted acquisition contracts: +IS_TIF filter; exact FILE_NAME acquisition metadata; METADATA pairing on +well/site/z_index/timepoint; per-alias CHANNEL component_identity. The failed +minimal example at 327cec9f6 remains in Git and the original a14 archive. +No production code changes after a14d471e3. The existing native handler does +not promise named source projection; do not infer ingestion owner from a +requested microscope enum or input occurrences from storage-plan rendering. + +engineering-acquisition-final-source: 6 PASS, 2.78s pytest / 3.50s wall / +334736 KiB RSS. Document parsing and synthetic path/declaration checks only; +no pixel load, initialization, compile, scientific execution or native startup. +engineering-source-bindings-ingress: initial 3 PASS / 1 FAIL, incorrect expected +exception class for an existing metadata conflict; original receipt retained. +engineering-source-bindings-identity-controls: 5 PASS, before final adoption +of the parent's precise complete authoring recipe. Source receipts remain +separate from the parent's installed21 and continuous installed MCP/native gate. +Parent reports compile/run complete, 8192 comparisons with maximum error +1.23e-7, exact declared calibration/first-source scope; no direct carrier/mask +inspection and no biology or R1 acceptance. See 344-engineering-acceptance.rst. + +New authoring receipts: docs/refactor/receipts/cp-aligned-full-stack-342-acquisition.tar.gz +and diagnostics/ACQUISITION_SHA256SUMS. Prior archives/manifests are unchanged; +their source hashes authenticate their recorded snapshots, not this revised example. + +Owner: H003g runtime repair worker under parent integration; draft PR #344. +H003g is permanently FAILED. These are synthetic source checks, not a trial +retry, installed acceptance, result QA or biological success evidence. + +Runner: run_source_tests.py. It prepends this worktree's Python source and +uses only the unchanged installed _tabular_native.abi3.so from +/home/ts/wt/openhcs-s1-installed-parent-20261001. Existing dependency packages +are reused without any download, build, install or configuration change. +All execution shards use CPU affinity 0, one CPU quota, 512 MiB MemoryMax, +zero swap, 60-second timeout, OPENHCS_CPU_ONLY=true, numerical threads 1, +and PYTEST_DISABLE_PLUGIN_AUTOLOAD=1. Two unknown asyncio configuration +warnings are expected because optional pytest plugins are disabled. + +Predecessor receipt interpretation (public 34203beb1) +--------------------------------------------------- + +merged-main-final-family-controls: 111 PASS, including 15 added full-stack +tests plus 96 existing executor/projection/alignment/device/registry controls. +merged-main-consumer-regressions: 45 PASS, including the unchanged-main ABI +fixture corrected using the real diagnostic declaration owner. +r0-merged-main-openhcs/scripts/benchmark: all PASS. Base main 7b0ec3f5, +head b7702664, exact three changed production paths. Subsequent edits affect +only tests/documentation/receipt packaging until the subsequent a14 correction. + +r1-original: import failure BEFORE scanning. The existing current NRA +environment does not export RedundantTypeCheckDetector at the original +guard's import site. Worktree external gitlinks are uninitialized as well. +No passing R1 certificate or complete dependency-context audit is claimed. +nra-before-corrected: deadline_incomplete structured report with null counts, +20-second CLI deadline, parsing unfinished. Its admitted candidate external +directory is uninitialized. This is NOT a successful no-findings report. + +Current source correction (a14d471e3) +------------------------------------ + +nominal-review-opaque-controls: 117 PASS (21 full-stack cases plus 96 existing +family controls), 3.36 seconds pytest, 322644 KiB RSS. New calibrated subtype, +real runtime-plane named bundle and opaque kwargs across all four contracts. +a14-consumer-controls: 45 PASS, 3.96 seconds pytest, 372732 KiB RSS. +a14-r0-openhcs/scripts/benchmark: exact original CI-pinned packaged ratchet, +base main 7b0ec3f5, head a14d471e3, all PASS. Root wall times 16.93/1.88/2.45 +seconds, RSS 88124/59020/57732 KiB. Optional-neutral nominal resolution is +the only additional production change after 34203beb1; opaque values keep +identity at FULL_STACK while slice-mode registration remains strict. + +engineering-document-separate-planes: 1 PASS, source authoring only, 1.95 +seconds pytest / 2.55 seconds wall / 277676 KiB RSS. The initial +decorator-wrapper object-identity failure and corrected +nominal/raw-callable identity receipt remain in the supplemental archive. +No compile, fixture pixels, MCP/native/viewer or installed check was run here. +The complete document and identity/path/metadata acceptance gates are under +docs/refactor/344-engineering-acceptance.rst and examples/. + +Parent independently retains the original installed opaque-kwargs failure; +those predecessor receipts are not relabelled. Parent owns the normal main +integration, paired rebuilt wheel and fresh installed/native numeric gate. +Original R1 remains unqualified for the previously recorded environment and +dependency-context limits; no new R1 completeness claim is made. + +Supplement: docs/refactor/receipts/cp-aligned-full-stack-342-a14-supplement.tar.gz. +NOMINAL_REVIEW_SHA256SUMS authenticates the current three production files, +test/document sources and new byte-preserved raw receipts. The original +cp-aligned-full-stack-342-20261001.tar.gz and SHA256SUMS are unchanged. + +Unsuccessful receipts remain in the archive: first-repair slotted-dataclass +super error; family-controls fixture argument and named bundle failure; +family-controls-corrected mask-domain failure; complete-owned-families +temporary metadata deletion error; frozen-executor-reproducer expected exact +float(AlignedImageStack) failure using original executor 20ca4f825 with empty +kwargs; provenance-contract-regressions stale diagnostic ABI fixture; +unchanged-main-abi-fixture confirms that failure on full unchanged main +0c0563e65; final-consumer-regressions nonexistent test paths/zero tests; +r0-original interpreter/dependency import failures; and nra-before incorrectly +admitted root. Earlier passing checkpoints are retained, not substituted for +the merged-main receipts. Logs contain exact original argv and time/RSS/exit +status, and JUnit files contain test identities and timestamps. + +Materialization scope +--------------------- + +The actual shared ABC algorithm owns pixels, masks and typed provenance. +Dense stacks, same-slice bundles, aligned runtime stacks and named output +bundles inherit it, with small axis/domain hooks. Existing nominal registry +dispatch adds full-stack projection for primary images and image kwargs; +other declared runtime families keep their own domains. No name switch, +np.asarray fix, mirrored registry/store or copied executor is shipped. + +The MRO cases use two independent test-only capabilities over the actual +composition owner: pixel offset and validity-mask creation. Both cooperative +super orders verify exact pixel and mask effects. They are not a claim of +production MI implementation or whole-package semantic equivalence. + +Artifact authority / bounded source replay +------------------------------------------ + +SHA256SUMS in this receipt archive covers all raw receipt files, the runner, +the five final changed Python files and the architectural source receipt. +Verify it after extracting into the reviewed source worktree. No science +input or trial output is included. Archived original stdout is byte-preserved; +whitespace within tracebacks is not normalized. + +Use the original runner and final log argv under the bounds above. To obtain +the expected pre-repair executor failure, set BASELINE_EXECUTOR_REF=20ca4f825 +and select only test_real_rescale_full_stack_materializes_composed_runtime_sources. +For a full unchanged-main baseline, git archive the recorded production and +fixture paths and set SOURCE_BASELINE_ROOT to the owned extracted directory. +No MCP/native/viewer startup is part of either source replay. + +R0 package: existing cached agent-comms package corresponding to original +CI pin 3b03785f45df2ef5dc62ba6aed99294192ecbb01. debt_ratchet.py SHA256 +e323c94d49c2b72d9524a5169f123e64b4a6e46a41035ca9fb4497e49b6ca562; +direct DeclaredFamily/FieldCodec/MroDispatch sources compare byte-identical. +The successful entrypoint uses /usr/bin/python3.14 and existing local +metaclass-registry source, matching the workflow interpreter, without install. +Original R1 source/script is unchanged; its environment failure is retained. + +Disposable scratch ownership +---------------------------- + +Only owned scratch h003g-runtime-repair-baseline-20261001 under +/home/ts/.cache/agent-scratch was used: 31 MiB unchanged-main source archive, +recoverable from Git. Its process completed and its failure logs are preserved +before cleanup. The attempted r1 scratch path was never created (import failed). +No native/viewer/MCP process, lifecycle lock or shared slot was acquired. +Parent owns installation/integration and actual synthetic installed acceptance +after #338. Held-out/reference data and frozen H003g remain sealed/unmodified. diff --git a/diagnostics/SHA256SUMS b/diagnostics/SHA256SUMS new file mode 100644 index 0000000000..c7c0c347ad --- /dev/null +++ b/diagnostics/SHA256SUMS @@ -0,0 +1,51 @@ +bd6a2291768984da8620e4ad12b2e15eb5017de95f5c5e74c175d2d35dbc6e87 diagnostics/complete-owned-families-corrected.log +bb22b22168aa5f1b93331d39f4bbdea068e8546a87accfd4412a47e97bd90b4e diagnostics/complete-owned-families.log +dec6e1ce77ce33abd98924a0263266415ee128bde87143797f7f7d19e1efb380 diagnostics/family-controls-corrected.log +bfff89adb6b1353253efa2e8238b01b598e098960ab8efc20401ca862cf5e26b diagnostics/family-controls.log +585c48a3e033ff3bb72ad30bc3414ee503c39983ea8449d82db31871303f1506 diagnostics/final-consumer-regressions-corrected.log +f6339b1ac8389a5db2849d540a7fc4f79a569ffe0b05acc4a1d834274f352f4f diagnostics/final-consumer-regressions.log +35da4640556585d845b3194a662d29e8e515976d1e9f8ed2dcaf87683e0bf389 diagnostics/final-owned-families.log +d222da0300ff48c16af1f194b550cc33f858e381c8163b0770cc3a20cfbab2ad diagnostics/first-repair.log +e267cbb0be15b211a486a00da973ef92b8ef8bd1bb14bddb6f3e78a6ba79e741 diagnostics/frozen-executor-reproducer.log +0967a18952104c4ddc7b70ce31633f87b3292bed95f27393597c5a8e9d9d19cc diagnostics/merged-main-consumer-regressions.log +cd33ccdf85417a5be1a0478969fd67159368388b4f9734f709692d2afa2bba0e diagnostics/merged-main-final-family-controls.log +cacb59314426b38bb686e4666387dfa601a164dff472035ab761d92395b068a9 diagnostics/merged-main-owned-families.log +1fcecfc03cde6e701f9b100202be7fc3c65162ed58577536506597abf658201c diagnostics/projection-regressions.log +a4fb0fb24d0123b156024ee2c7009d52f55cbc3f15692f328658bb0ea699fe0c diagnostics/provenance-contract-regressions.log +6ff216a32c668e1de28c038423cd761393d2feea932fa36561aa4e0939e3594f diagnostics/r0-merged-main-benchmark.log +6683a0fa686780933dc4d4409e2caa8848ea76c08045244f60d6332ec68f38f7 diagnostics/r0-merged-main-openhcs.log +3965ee524e244721bcc9d2a9afcc215ed8d5b420c110ebbd6e7f08b04baa62fd diagnostics/r0-merged-main-scripts.log +5ad08ecc61b036eeab23028811cd00a1cbc91284e9e99d33cf3c0a7cb04664c0 diagnostics/r0-original-openhcs-corrected.log +bf0589d6dc42c2808b8a2e864f3b84238970ddddbc47aa27e9e8fc4702e5cb05 diagnostics/r0-original-openhcs-python314.log +0894c154c887d3d2373e269d4f073f91aab5727709716f5e4f3df59dc7708478 diagnostics/r0-original-openhcs.log +cbf59c2358186f77d602ec8fd33667db06efe030c7f7ad847a5b80785b7531e8 diagnostics/r1-original.log +d79282c4ec609eddfb9e7f2bb9c38114f520238b0a3b192d0766edddbfb91e5e diagnostics/unchanged-main-abi-fixture.log +6376d12a0fb269bdd1636cfbcb73b8f877eac9ad53b6003b936fe13bbf84332b diagnostics/working-checkpoint.log +39e966c857dc5134b1c5c7ec0b02c057de5b9758399aee13f0048e6e542587e1 diagnostics/complete-owned-families-corrected.xml +7c8a59a60af25c33f5c0314fd00b8746a832b43451faacf0f47f289e4b4437cc diagnostics/complete-owned-families.xml +99f496dff5d8f503dba8b5d07951f08634139d964e0e6482f1321715cda935a1 diagnostics/family-controls-corrected.xml +f1436c71ddecdbf407a4f32b7eca00203c669e5f8b6c604527036c3469a49cde diagnostics/family-controls.xml +b8ffc53f650e0c115dc3026351038dc8473c67a3ec7c00af3f1247ed5689d02a diagnostics/final-consumer-regressions-corrected.xml +b149d129564ba311cbe21a09351a1a1ce312e61b385d0c29bcaf218f83263fe8 diagnostics/final-consumer-regressions.xml +3ec553b2c8903d239cc480cdc837446300af3ba7d1700cc4f57dfe8bf7f4672a diagnostics/final-owned-families.xml +e69cf0f62a1cb2cee00ceb686d08f2fd90d14a94bd9f2460dc5da55e5e7448e6 diagnostics/first-repair.xml +44da22ae325258524ecd39e3217d34e9043ac4aa785c5990e18297f02f8ef736 diagnostics/frozen-executor-reproducer.xml +f7e62750a3ca5012547a6da8580a7c4dc77f9c5ab08c733cc9a947805cfd25fe diagnostics/merged-main-consumer-regressions.xml +183d6ddc5333dfcd453a7140db3f40c92e4550fd54369f40ca20c1c81239529f diagnostics/merged-main-final-family-controls.xml +b7b762e000d244910b56f7dcaa40e088c1289f8d9c6d59d827a3e295ed3e9759 diagnostics/merged-main-owned-families.xml +70f3c819b91532672658f2241678e3aa6039d38404046407e68722129fb825b4 diagnostics/projection-regressions.xml +f08aa57d01f271edea64cc82a91a005ad6887609b06b430d3232067f17ef8042 diagnostics/provenance-contract-regressions.xml +e9f46720354d284abac14390b74256f1ab043fd7b260765663d6e0b9f24deeb0 diagnostics/unchanged-main-abi-fixture.xml +46cb6c97933d1875a901a28b9aa08672ab787c11c8083e115abe33cc9c01ad40 diagnostics/working-checkpoint.xml +e1040c5f2bf64b1a4ca3c26a143c24cb7b6c316e220843c6b24167838d15ee22 diagnostics/nra-before-corrected.json +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diagnostics/nra-before.json +553f21287efe2f9ef9e284dcb2f0c258c3f8b4d8bbf1880b155638d73b524d27 diagnostics/nra-before-corrected.stderr +d3234db1a85608fecaf5997857ac58569945273c5f06a782904a06ea14db311b diagnostics/nra-before.stderr +2dd04e5005b9cde8b1b9b86f0d17e3b9c45120795e9be0ed9b556787b33ffdf2 diagnostics/run_source_tests.py +047382c772441e3287ca8de8dc6db1ea269b6e02b7fca4ebed6da676a1aae34b diagnostics/README.rst +3514ad8f2346e5b5e82aaf4410610afc2fc8616eaf4596d5af1eaa387a85efe5 openhcs/core/aligned_image_payload.py +504be26040871a58c127488bb9a1536ff30a6e538b5234a69a6c216c0030371f openhcs/core/runtime_slice_projection.py +51fa66703e1d789c199ba485f80ec71152bb1f64a3864ed2414d167660ffc2f8 openhcs/interop/cellprofiler/runtime/function_contract_execution.py +c233aa9a9a49dbf0d39ccde858766056dd07d8fa0fdd8677dd4dc6804f17ee84 tests/unit/test_cellprofiler_full_stack_materialization.py +72918563a6e410956e41ba902ef2affb4fae35f60169c924dcabb233715c6e0a tests/unit/test_cellprofiler_runtime_input_binding_contract.py +489111c6b9dcb022ae452a072d3fa74dc287866387ea101578a81a36cdafc023 docs/refactor/cp-aligned-full-stack-342.rst diff --git a/diagnostics/run_source_tests.py b/diagnostics/run_source_tests.py new file mode 100644 index 0000000000..4f7a443623 --- /dev/null +++ b/diagnostics/run_source_tests.py @@ -0,0 +1,37 @@ +"""Run owned source checks using only the unchanged installed native artifact.""" + +import os +import subprocess +import sys +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +root = Path(os.environ.get("SOURCE_BASELINE_ROOT", str(root))).resolve() +sys.path.insert(0, str(root)) +import openhcs.core + +artifact_root = Path( + "/home/ts/wt/openhcs-s1-installed-parent-20261001/.venv/lib/python3.12/site-packages/openhcs/core" +) +openhcs.core.__path__.append(str(artifact_root)) +print("source root:", root, flush=True) +print("native artifact:", artifact_root / "_tabular_native.abi3.so", flush=True) +baseline_ref = os.environ.get("BASELINE_EXECUTOR_REF") +if baseline_ref: + import openhcs.interop.cellprofiler.runtime.function_contract_execution as executor_module + + source = subprocess.check_output( + [ + "git", "show", + f"{baseline_ref}:openhcs/interop/cellprofiler/runtime/function_contract_execution.py", + ], + cwd=root, + ) + print("baseline executor ref:", baseline_ref, flush=True) + exec( + compile(source, f"git:{baseline_ref}:function_contract_execution.py", "exec"), + executor_module.__dict__, + ) +import pytest + +raise SystemExit(pytest.main(sys.argv[1:])) diff --git a/diagnostics/test_344_engineering_document.py b/diagnostics/test_344_engineering_document.py new file mode 100644 index 0000000000..554e5c5ecc --- /dev/null +++ b/diagnostics/test_344_engineering_document.py @@ -0,0 +1,60 @@ +"""Source-authoring checks only: no data loading, compile, execution or processes.""" + +from pathlib import Path + +from openhcs.constants import AllComponents, GroupBy, Microscope, VariableComponents +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.artifacts import ArtifactInputPlan, ArtifactOutputPlan, ImageArtifactType +from openhcs.core.callable_contract import CallableContract +from openhcs.core.pipeline_document import PipelineDocumentAuthority +from openhcs.interop.cellprofiler.module_declarations import CellProfilerModule +from openhcs.processing.backends.cellprofiler.intensity import ( + RescaleIntensityModule, + rescale_intensity, +) +from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract + + +def test_complete_engineering_document_uses_declared_primary_sources(): + source = ( + Path(__file__).resolve().parents[1] + / "docs/refactor/examples/344-aligned-rescale-engineering.py" + ).read_text() + document = PipelineDocumentAuthority.from_source(source) + assert document.original_source == source + assert document.pipeline_config.microscope is Microscope.IMAGEXPRESS + plan = document.pipeline_config.source_bindings_config + # Each fixture file is 2-D. Z varies across files, not inside each TIFF. + assert plan.source_stack_components == () + assert plan.source_voxel_spacing.values_zyx == (4.0, 0.5, 0.5) + assert tuple(binding.alias for binding in plan.bindings) == ( + "EngineeringCH1", "EngineeringCH2", + ) + assert tuple(binding.selector.components[0].value for binding in plan.bindings) == ( + "1", "2", + ) + (step,) = document.pipeline_steps + assert step.processing_config.variable_components == [VariableComponents.Z_INDEX] + assert step.processing_config.group_by is GroupBy.NONE + assert step.source_bindings.enabled + func, kwargs = step.func + # The document authority canonicalizes decorator callables via RegistryService. + contract = CallableContract.from_callable(func) + assert CellProfilerModule.require_callable_contract_owner(contract) is RescaleIntensityModule + assert contract.resolve_canonical_raw_callable() is ( + CallableContract.from_callable(rescale_intensity).resolve_canonical_raw_callable() + ) + bindings = RescaleIntensityModule.declared_artifact_bindings( + plan_type=ArtifactInputPlan, artifact_type=ImageArtifactType, + ) + assert len(bindings) == 2 + assert all(binding.runtime_parameter_name is None for binding in bindings) + assert tuple(kwargs[binding.require_parameter_name()] for binding in bindings) == ( + "EngineeringCH1", "EngineeringCH2", + ) + (output,) = RescaleIntensityModule.declared_artifact_bindings( + plan_type=ArtifactOutputPlan, artifact_type=ImageArtifactType, + ) + assert kwargs[output.require_parameter_name()] == "EngineeringRescaled" + assert contract.runtime_image_execution_mode is ImagePayloadExecutionMode.FULL_STACK + assert contract.metadata.processing_contract is ProcessingContract.PURE_2D diff --git a/diagnostics/test_344_source_binding_ingress.py b/diagnostics/test_344_source_binding_ingress.py new file mode 100644 index 0000000000..489c1c5297 --- /dev/null +++ b/diagnostics/test_344_source_binding_ingress.py @@ -0,0 +1,135 @@ +"""Pure path/declaration reproducer: no pixels, workspace writes or native calls.""" + +from dataclasses import replace +from pathlib import Path + +import pytest +from polystore.base import _create_storage_registry +from polystore.filemanager import FileManager + +from openhcs.constants import AllComponents, Backend, Microscope +from openhcs.core.pipeline_document import PipelineDocumentAuthority +from openhcs.core.source_binding_workspace import SourceBindingWorkspaceProjector, SourceSetAssembler +from openhcs.core.source_bindings import ( + MetadataExtractionRule, MetadataSource, source_bindings_defaults_to_base, +) +from openhcs.core.source_projection import OpenHCSPlaneAddress, SourceProjectionSet +from openhcs.microscopes.microscope_base import create_microscope_handler +from openhcs.microscopes.source_bindings_handler import SourceBindingsHandler + + +EXAMPLE = ( + Path(__file__).resolve().parents[1] + / "docs/refactor/examples/344-aligned-rescale-engineering.py" +) +ROOT = Path("/synthetic/engineering-fixture") +PATHS = tuple( + ROOT / f"TimePoint_1/ZStep_{z}/A01_s{site:03}_w{channel}_z{z:03}_t001.tif" + for z in (1, 2) for site in (1, 2) for channel in (1, 2) +) + + +def _config(): + document = PipelineDocumentAuthority.from_source(EXAMPLE.read_text()) + assert document.pipeline_config.microscope is Microscope.IMAGEXPRESS + return source_bindings_defaults_to_base(document.pipeline_config.source_bindings_config) + + +def test_explicit_imagexpress_with_bindings_is_routed_to_generic_ingestion(): + # Factory construction only. No initialize_workspace, I/O or compile. + handler = create_microscope_handler( + microscope_type=Microscope.IMAGEXPRESS.value, + filemanager=FileManager(_create_storage_registry()), + source_bindings_config=_config(), + ) + assert isinstance(handler, SourceBindingsHandler) + + +def test_metadata_empty_candidates_reproduce_exact_order_guard(): + config = replace(_config(), metadata_rules=(), match_plan=None) + projector = SourceBindingWorkspaceProjector(config) + candidates = projector.source_candidates(ROOT, PATHS, source_backend=Backend.DISK) + selected = projector._candidates_by_alias(ROOT, candidates, filemanager=None) + refs = tuple(tuple(item.source_ref for item in values) for values in selected.values()) + assert len(refs) == 2 and refs[0] == refs[1] and len(refs[0]) == 8 + with pytest.raises(ValueError, match="ORDER source bindings must distinguish aliases"): + SourceSetAssembler.for_config(config).source_sets(None, selected, ()) + + +def test_declared_acquisition_metadata_distinguishes_channels_site_and_z(): + config = _config() + projector = SourceBindingWorkspaceProjector(config) + candidates = projector.source_candidates(ROOT, PATHS, source_backend=Backend.DISK) + assert len(candidates) == 8 + selected = projector._candidates_by_alias(ROOT, candidates, filemanager=None) + assert tuple(selected) == ("EngineeringCH1", "EngineeringCH2") + assert tuple(len(values) for values in selected.values()) == (2, 2) + for channel, alias in enumerate(selected, start=1): + members = selected[alias] + assert tuple(member.metadata["z_index"] for member in members) == ("1", "2") + assert all(member.metadata["well"] == "A01" for member in members) + assert all(member.metadata["site"] == "1" for member in members) + assert all(member.metadata["timepoint"] == "1" for member in members) + assert all(member.metadata["channel"] == str(channel) for member in members) + assert tuple(member.source_ref.backend_address for member in members) == tuple( + f"TimePoint_1/ZStep_{z}/A01_s001_w{channel}_z{z:03}_t001.tif" for z in (1, 2) + ) + sets = SourceSetAssembler.for_config(config).source_sets(config.match_plan, selected, ()) + assert len(sets) == 2 + for source_set in sets: + members = tuple(source_set.candidates_by_alias.values()) + assert members[0].source_ref != members[1].source_ref + assert members[0].metadata["z_index"] == members[1].metadata["z_index"] + + +def test_declared_folder_and_filename_z_conflicts_remain_rejected(): + config = _config() + # The shipped recipe declares filename identity; this extra rule is a conflict + # control of the existing extractor, not an extra required fixture declaration. + config = replace(config, metadata_rules=(*config.metadata_rules, MetadataExtractionRule( + MetadataSource.FOLDER_NAME, + r"(?:^|/)ZStep_0*(?P[1-9][0-9]*)$", + ))) + projector = SourceBindingWorkspaceProjector(config) + mismatched = ROOT / "TimePoint_1/ZStep_2/A01_s001_w1_z001_t001.tif" + with pytest.raises(RuntimeError, match="Conflicting metadata field 'z_index'"): + projector.source_candidates(ROOT, (mismatched,), source_backend=Backend.DISK) + + +def test_store_singleton_addresses_need_declared_component_identity(): + config = _config() + projector = SourceBindingWorkspaceProjector(config) + candidates = projector.source_candidates(ROOT, PATHS, source_backend=Backend.DISK) + # This reproduces the parent's stated store-plane boundary without reading TIFFs. + singleton = OpenHCSPlaneAddress.from_values( + well="A01", site="1", channel="1", z_index="1", timepoint="1", + ) + candidates = tuple(replace(item, declared_address=singleton) for item in candidates) + metadata_only = replace(config, bindings=tuple( + replace(binding, component_identity=()) for binding in config.bindings + )) + old_projector = SourceBindingWorkspaceProjector(metadata_only) + old_selected = old_projector._candidates_by_alias(ROOT, candidates, filemanager=None) + old_sets = SourceSetAssembler.for_config(metadata_only).source_sets( + metadata_only.match_plan, old_selected, (), + ) + with pytest.raises(ValueError, match="Duplicate source projection address"): + SourceProjectionSet(old_projector._source_set_projections(metadata_only.bindings, old_sets)) + + selected = projector._candidates_by_alias(ROOT, candidates, filemanager=None) + sets = SourceSetAssembler.for_config(config).source_sets(config.match_plan, selected, ()) + projections = SourceProjectionSet(projector._source_set_projections(config.bindings, sets)) + assert len(projections.projections) == 4 + assert tuple(( + projection.address.value_for(AllComponents.WELL), + projection.address.value_for(AllComponents.SITE), + projection.address.value_for(AllComponents.CHANNEL), + projection.address.value_for(AllComponents.Z_INDEX), + projection.address.value_for(AllComponents.TIMEPOINT), + ) for projection in projections.projections) == ( + ("A01", "1", "1", "1", "1"), ("A01", "1", "2", "1", "1"), + ("A01", "1", "1", "2", "1"), ("A01", "1", "2", "2", "1"), + ) + assert {projection.ref for projection in projections.projections} == { + item.source_ref for members in selected.values() for item in members + } diff --git a/docs/refactor/344-engineering-acceptance.rst b/docs/refactor/344-engineering-acceptance.rst new file mode 100644 index 0000000000..2556bd77e9 --- /dev/null +++ b/docs/refactor/344-engineering-acceptance.rst @@ -0,0 +1,146 @@ +PR344 engineering document and acceptance disposition +==================================================== + +Runtime source checkpoint: a14d471e3f2a92148f88a60b6c972e1a49f67db0. +Parent integrated current main and owns installed/native acceptance. Later +child checkpoints change only authoring examples, diagnostic checks and receipts. +H003g remains permanently FAILED and untouched. This is engineering, not biology. + +Corrected complete source +------------------------- + +examples/344-aligned-rescale-engineering.py is one complete PipelineDocument +with pipeline_config and one ordinary registered RescaleIntensity FunctionStep. +Its imports and declaration-derived input/output keyword names are explicit. +It contains no custom callable, executor wrapper, arrays or pixel-file operations. + +The current recipe adopts the parent's successfully admitted private engineering +document's acquisition declarations, with only this example's output root/header +different. Submit the entire source via normal MCP authoring/compile/run. +Never strip its pipeline source bindings or substitute another config argument. + +Separate fixture: the parent's cp-stack-installed-20261001/fixture contains +eight 2-D TIFFs, sites 1/2, channels 1/2, Z 1/2, timepoint 1. Representative +physical spelling:: + + TimePoint_1/ZStep_1/A01_s001_w1_z001_t001.tif + TimePoint_1/ZStep_2/A01_s001_w2_z002_t001.tif + +Required existing declarations: + +* source_filters EXTENSION / IS_TIF excludes HTD and other sidecars. +* FILE_NAME MetadataExtractionRule extracts well/site/channel/z_index/timepoint + from the actual verified filename grammar. Zero padding is excluded from the + numeric capture, so s001 resolves site "1", not a new site "001". +* METADATA SourceBindingMatchPlan explicitly pairs both aliases by + well/site/z_index/timepoint. Channel distinguishes aliases, not the pair key. +* Both typed selectors restrict SITE "1" and their respective CHANNEL. +* Each NamedSourceBinding.component_identity explicitly declares CHANNEL "1" + or "2", selecting the documented semantic coordinate projection. +* Calibration is (4.0, 0.5, 0.5) micrometers Z/Y/X. There are no embedded stack + axes in a 2-D file, so source_stack_components remains empty. The step declares + variable Z_INDEX, GroupBy.NONE and enabled inherited source bindings. + +Source diagnosis and ownership +------------------------------- + +Inspected source: public 327cec9f6 and parent candidate c7ed2acc4. Their +source_binding_workspace.py is byte-identical. SourceBindingWorkspaceProjector +source_candidates (line 886) builds file candidates from declared extraction +rules, not an injected native filename parser. _component_is_compatible permits +absent coordinates (line 997); _candidate_with_binding_components then assigns +selector coordinates. Without acquisition metadata, both aliases therefore +select the same physical universe; the ORDER ambiguity guard correctly rejects it. + +ImageXpress inherits MicroscopeHandler.projects_declared_source_bindings False. +The current create_microscope_handler factory selects SourceBindingsHandler for +nonempty bindings when the requested owner does not project them. Raw inspection +without bindings can retain ImageXpress and parse its native coordinates. Thus +IMAGEXPRESS in pipeline source alone does not certify the resolved ingestion owner. +The child construction-only test verifies the factory route, not the parent's +live handler class; preserve actual handler identity in the parent receipt. + +For store-emitted candidates, _source_set_projections (line 1157) retains +declared_address unless component_identity is explicitly declared. Metadata +extraction alone does not override a store's singleton address. NamedSourceBinding +documents component_identity as authoritative over inferred store coordinates +(source_bindings.py line 867). Explicit channel identity invokes the existing +coordinate projection; matched metadata supplies well/site/Z/time. + +Disposition: the minimal public recipe omitted required acquisition declarations. +It was not a compiled valid native-alias recipe. Its inference of native alias +projection was wrong. The existing native handler does not promise that capability; +automatic propagation is a product/ingestion limitation, not a proven regression +of a promised alias-projection contract. No production intake, selector, guard, +dispatcher or executor changes are made. NRA BOUND-2 guidance keeps identity with +the existing metadata/binding/coordinate owners rather than adding a second parser. + +Predecessors are not relabelled as passes. Parent reports four original admission +attempts retained, including same-ref ORDER rejection, duplicate projection address, +and HTD lacking well identity. Public predecessor source survives at 327cec9f6 +and inside the original a14 supplemental archive. The promoted example now carries +the complete admitted acquisition declarations rather than the invalid minimal form. + +Plans, occurrence evidence and output scope +------------------------------------------- + +Expected projected source inventory: A01/site1/time1, four virtual files +CH1/2 x Z1/2, with exact physical refs and calibration. Both RescaleIntensity +declaration inputs are primary ImageArtifactType occurrences with no runtime +parameter injection. Callable mode is FULL_STACK, contract PURE_2D. + +The public artifact-plan artifact_inputs list is not the complete input-occurrence +surface: _bounded_step_summaries renders step_plan.artifact_inputs storage plans. +PathPlanner skips a storage input plan when an exact source binding satisfies it +(path_planner.py line 1817). Exact invocation occurrences instead live in +CompiledFunctionInvocation.artifact_input_edges and contract.artifact_inputs. +RuntimeInputBindingRequest selects those declarations through RuntimeAdapterRequest, +then resolves a source-satisfied edge via source_binding_plan/source_artifact_payload. + +For direct carrier proof, inspect those exact selected occurrences, their two +source bindings, runtime source payload axes and composed primary value before +the FULL_STACK projection. An empty storage-plan list does not prove missing +bindings, and a successful multi-plane output does not prove the carrier's type. +The actual MCP surface did not expose carrier or validity-mask inspection. + +EngineeringRescaled declares SourceStackLineageSourceRelation to EngineeringCH1. +Output source identity must follow that first-source lineage, not a fabricated +four-source output contract. Output plate root for this published example:: + + /home/ts/wt/openhcs-344-engineering-20261001/output/fixture_openhcs + +For plate name fixture, well A01, step0, no grouping key, the canonical runtime +artifact-store address is:: + + results/A01_EngineeringRescaled_step0.pkl + +This is not a promised durable pickle. Compile/materialization plans own actual +TIFF destinations; retain the descriptors rather than guessing filenames. +The earlier EngineeringPlate example root changes if that is the input basename. + +Bounded source checks and parent acceptance +------------------------------------------- + +Child source-only checks parse the complete document and verify registered owner/ +canonical raw callable identity. Pure synthetic path/declaration checks reproduce +the ambiguity and singleton-address failures, then prove four unique coordinates +with the same physical SourcePixelRefs. No TIFF pixels are loaded, no workspace is +initialized, and no pipeline is compiled/executed by this child. A first conflict +test used the wrong exception class; its 3-pass/1-fail receipt is retained, and the +corrected assertion preserves the actual existing RuntimeError contract. + +Parent's reported installed/native gate (2026-10-01): same owned native5993, +ordinary MCP compile job1 COMPLETE in 1.823s; execution job2 COMPLETE in 0.745s. +Main-flow and EngineeringRescaled each yielded two Z TIFFs, each (2,32,32) float32. +MCP samples were compared against all four raw planes with global stretch +min90/max8405: 8192 pixel comparisons across both destinations, maximum error +1.23e-7 and zero errors over 1e-6. Parent reports exact declared calibration and +first-source CH1/Z scope, and retains the actual native observation export. +These are parent observations, not independently rerun child measurements. + +This clears the reported installed dense ABI continuity gate, not direct carrier/ +mask inspection or biology. Raw TIFF validity masks are None; source MaskedImagePayload +and capability-MRO controls are separate evidence. Child source117+45 and parent's +revised installed21 cases remain distinct from this continuous MCP/native gate. +R1 remains unqualified. Parent alone closes/verifies owned processes, archives live +receipts and integrates the public PR. No frozen H003g retry or held-out access occurs. diff --git a/docs/refactor/344-installed-native-acceptance.rst b/docs/refactor/344-installed-native-acceptance.rst new file mode 100644 index 0000000000..af0c7516cd --- /dev/null +++ b/docs/refactor/344-installed-native-acceptance.rst @@ -0,0 +1,123 @@ +PR344 installed native engineering acceptance +============================================ + +Integration owner: parent. Source owner: Darwin. Runtime source a14d471e3; +paired private integration c7ed2acc4 includes main 05c3cf288. Documentation-only +327cec9f6 was normally merged afterward at 29e08a1c9. All three changed +production files are byte-identical to the installed wheel and a14 source. +No worker worktree, frozen trial, user desktop or global package was modified. +The example correction a422c4f08 changes only authoring/diagnostics/receipts, +not the installed production code. + +BOUND-2 / IMPL-12 / IMPL-13: existing ImagePayloadStackComposition owns the +shared pixel/mask/provenance algorithm; nominal projection hooks materialize +aligned primary and auxiliary images. Cooperative capability MRO checks are +source/installed tests, not a new production mixin-family claim. The original +R0 source guard passes; full NRA/R1 remains unqualified for the reasons in +cp-aligned-full-stack-342.rst. This gate does not clear that missing audit. + +Installed candidate +------------------- + +Private interpreter: /home/ts/wt/openhcs-cp-stack-installed-parent-20261001/.venv. +Revised wheel SHA256: +5e4fc51f8d6c28ed73ae9dc1a3ac68126eee37cf0ddb7e37e797694ca29a8f0e. +Build: PASS, 7.32 seconds, 249.61 MiB combined RSS, one CPU/512 MiB/60 seconds. +Actual installed numeric controls: 21 PASS, 2.81 seconds wall, 269.38 MiB RSS. +These include opaque kwargs, calibrated subtype, exact pixels/masks/provenance, +named source bundles and independent capability composition in both MRO orders. + +Original installed predecessor and failures remain separately retained: its +15 numeric cases passed, but the additional opaque-kwarg check failed with +RuntimeSliceProjectionDeclarationError. The first pytest invocation failed +before collection because of a repository plugin; the isolated invocation +disables repo-only configuration without weakening tests. No failed receipt +was overwritten or relabelled as revised acceptance. + +Continuous installed MCP/native journey +-------------------------------------- + +Engineering root: +/home/ts/wt/openhcs-issue-batch-20260929/cp-stack-installed-20261001. +Actual installed dev-client shell used ordinary 10-second observation limits. +Health was current, packaged resources ready, no source drift. CPU-only owned +TCP 127.0.0.1:5993 native process 918063/create_time 1790840817.41 was started +ONCE through MCP, observed by its complete handle and retained through cold +capability preparation until READY. Function catalogue preparation then reported +READY on that same incarnation. No retry/restart on observation expiration. + +MCP generated a fresh ImageXpress fixture: eight 32x32 uint16 files, two sites, +two channels, two Z planes, one A01/timepoint. The complete PipelineDocument +selects site 1 only, two named primary sources, Z_INDEX, GroupBy.NONE and the +ordinary registered CellProfiler RescaleIntensity callable. No direct array +executor, custom wrapper, H003g data or scientific parameter reuse was involved. + +The initial engineering document was incomplete at acquisition ingress. Three +failed artifact-plan admissions are retained in order: identical ORDER source +references, duplicate projection address, then unfiltered HTD missing well +metadata. Correct authoring uses existing typed authorities: TIFF source filter; +explicit filename metadata extraction; METADATA pairing by well/site/Z/time; +explicit channel component_identity. The ambiguity guard was never bypassed. +These source declarations distinguish selection, acquisition identity and source +pairing; they are not processing-function filename parsing or library patches. +The requested IMAGEXPRESS enum is not proof of the effective alias-ingestion +owner. Its handler does not project declared bindings; the existing factory +selects SourceBindingsHandler for this document. Native execution here means +the real installed execution server, not a claimed native-vendor alias handler. +The original minimal recipe was incomplete; no promised ingestion regression +was demonstrated and no production source-selection fix was introduced. + +The final artifact plan has exactly four distinct physical planes and aliases +EngineeringCH1/EngineeringCH2 across Z1/Z2, all at site1/A01/timepoint1. Source +metadata retains exact physical paths and explicitly declared synthetic physical +spacing (4,0.5,0.5) micrometers. This override is an engineering metadata check, +not a measured physical calibration of the generated assay. + +Headless session session-1: compile job-1 / aa473313-f45c-478c-bac1-103d42acd9d9 +COMPLETE in 1.823 seconds; execution job-2 / +3a9bf8c4-dec9-4960-b25c-db5fcae07499 COMPLETE in 0.745 seconds. Execution also +recompiled through the ordinary route; no compile-cache speed claim. Runtime +observation was exported through MCP and retained without an external image +decoder. Native step compute was 0.035 seconds. No UI/viewer was launched. + +Output verification and limits +------------------------------ + +MCP inspected/query/sampled the actual output plate. Both main-flow and named +EngineeringRescaled destinations contain two Z-position TIFFs; each is a +float32 (2,32,32) source-binding bundle. Exact full-resolution MCP samples of +the four raw planes and all four output files compare every output pixel to +(input-90)/(8405-90). Across 8192 comparisons the maximum absolute error is +1.2279345340537873e-7; zero exceed 1e-6. Both bindings and both Z positions are +numerically exercised, not merely nonzero/extrema checked. + +Output metadata correctly follows the declared FIRST-source lineage: CH1 alias, +Z1/Z2, original source paths and declared spacing survive. This is not a new +all-contributor output-provenance contract. Input plan inspection has four +source-workspace entries but artifact_inputs is empty for bound sources; the +surface does not directly expose the composed invocation carrier. Accordingly +this receipt does not claim direct native carrier or nontrivial validity-mask +inspection. Those nominal/mask witnesses are separate source/installed controls. +Output inspection reports its missing grid_dimensions warning; it is retained, +not hidden by successful pixel comparisons. No biological completeness, counts, +held-out generalization or segmentation QA is inferred. + +Same-handle MCP force close acknowledged, endpoint terminated, process_exited +true. Parent confirmed MCP/client/native PIDs absent and 5993/6993 listeners +gone; the shell's exit status1 is retained, not reported as a clean CLI exit. +No foreign endpoint or user's isolated H002 viewer was touched. + +verify_mcp_acceptance.py replays only recorded JSON payloads: PASS, 0.15 seconds, +15.48 MiB RSS. It asserts all three original admission failures, final source +identity/calibration, exact job scope, all pixel values and exact owned close. +Archive includes raw transcript/input/timing, fixture/output, complete document, +runtime observation, installed JUnit/build/failure logs and verifier. Wheel files +remain local and are identified by hash; no redundant binary wheel is in Git. +Archive: receipts/cp-stack-installed-native-344-20261001.tar.gz (199971 bytes), +SHA256 d6d53528b2a8508f4004a6d1d6b7adfc411a7dea9a1ed1844bb19736731faad5. +Fresh extraction reran the recorded-payload verifier successfully and compared +the extracted transcript byte-for-byte to its original. + +Remaining: full NRA/R1 disposition, wider S1/ZIP closure, updated paired install +and a NEW bounded context-isolated blind run with distributed matched visual QA. +Frozen H003g remains FAILED; this is neither its replay nor biological acceptance. diff --git a/docs/refactor/cp-aligned-full-stack-342.rst b/docs/refactor/cp-aligned-full-stack-342.rst new file mode 100644 index 0000000000..f0aacbf692 --- /dev/null +++ b/docs/refactor/cp-aligned-full-stack-342.rst @@ -0,0 +1,226 @@ +CellProfiler aligned full-stack boundary (#342) +============================================== + +Owner: H003g runtime repair worker, under parent integration. +Audited base: OpenHCS main 0c0563e6538af313345391f2dc314270a86933a0. +H003g remains a frozen FAILED development repeat. No scientific retry or +native/MCP/viewer startup is authorized by this repair. + +Diagnosis +--------- + +``CellProfilerModuleExecutor._image_request`` composes source/artifact image +values through ``compose_aligned_image_payload``. Multiple runtime-aligned +inputs produce an ``AlignedImageStack`` with one composed image bundle per +runtime slice. ``_invocation_request`` then applies the callable-owned +``runtime_image_execution_mode``. A FULL_STACK declaration supersedes the +composition's ALIGNED_MULTI_IMAGE_STACK mode without materializing its payload. +``CellProfilerFunctionContractExecutor.execute_pure_3d`` passes this alignment +carrier straight to the raw callable, whose numeric array contract is violated. + +The retained candidate SHA256 is +7b69ce1f4256e2a64b32ecb1fefd6d22adef7ba3a1d98f5e2a3fc0125eab96b3; +the retained native log identifier is 1790835483807054701. The first failing +call is rescale_intensity, at RescaleIntensityContext.from_settings, +``source_data.astype(np.float32, copy=False)``: float(AlignedImageStack). + +The synthetic two-runtime-plane/two-input composition reproduces this exact +stack with EMPTY kwargs: the primary image is the immediate root, not an +auxiliary binding. It does not read trial images or scientific outputs. Aligned +image kwargs have the same unmaterialized boundary and are covered separately. +The executor's +unchanged contract suite passed 32 cases while this added case failed. +The raw full-stack pass-through exists in 9110b10479 (June 2026); PR326 changes +neither this executor nor aligned_image_payload. This is an existing uncovered +boundary, not demonstrated to be a PR326 regression. Historical #204 is the +separate selector/input-scope surface and does not supply a fix here. + +Ownership and migration +----------------------- + +BOUND-2 and IMPL-12/13: the existing pixel stacking, mask composition and typed +provenance authorities are reused, not copied into the CellProfiler executor. +``ImagePayloadStackComposition`` is the actual owning ABC/template algorithm. +The existing same-slice ``ImagePayloadBundleContext`` inherits it through the +explicit dense-stack context, retaining its channel/spatial-mask hooks. +``AlignedImageStack`` inherits the same algorithm with runtime-slice inputs and +contributor provenance hooks. ``ImageOutputBundle`` remains a distinct named +source-binding axis; it is not treated as anonymous runtime alignment. + +The existing AutoRegisterMeta/MRO-owned RuntimeSliceProjectionStrategy family +supplies the full-stack projection hook. The aligned image leaf calls the +payload owner; every other nominal family retains its own value. The executor +uses this authority for the primary image and kwargs before the raw call. +Object labels, measurement rows, graphs and RuntimeSliceAlignedValues are not +flattened into guessed arrays. The existing PURE_3D rejection of slice-aligned +non-image kwargs remains active. No callable-name branch, second registry, +numeric wrapper, alternate executor or installation change is introduced. + +Source checkpoints +------------------ + +33 source cases passed, 2.35 seconds pytest / 3.08 seconds wall, +305672 KiB RSS. One CPU and a 512 MiB cgroup with no swap, 60-second timeout. +The isolated installed interpreter supplies existing dependencies; the source +runner resolves this worktree's Python code and only the unchanged installed +``_tabular_native.abi3.so`` artifact. No download/build/install is performed. + +The first complete-package NRA attempt rejected a misdeclared context root. +The corrected whole-package scan admitted the installed candidate's external +directory, but that candidate's submodules are uninitialized; this is not +complete dependency context. The CLI's own 20-second deadline expired while +parsing (outer shard bound 60 seconds), with 511664 KiB RSS. +No complete global semantic/proof certificate +is claimed. Manual ownership evidence and source behavior tests are distinct. +The initial slotted-dataclass zero-argument super failure is retained and fixed +with explicit cooperative super(AlignedImageStack, self), not suppressed. + +Expanded checkpoint: 110 cases PASS, 3.38 seconds pytest / 4.19 seconds wall, +322736 KiB RSS, under the same one CPU / 512 MiB / no swap / 60-second bound. +The 14 new cases cover the real rescale callable, all four ProcessingContract +families with singleton and two-slice image/auxiliary inputs, exact contributor +identities and slice projections, voxel calibration, masked source-binding +planes composed into a runtime stack, named output bundles, ragged-input +rejection before dispatch, and independent pixel/mask capabilities composed in +both cooperative MRO orders. The other 96 are unchanged executor and nominal +projection/alignment/device/registry controls. The named bundle strategy now +inherits the aligned-image strategy's full-stack hook; its own source-binding +projection/count behavior stays separate. Shared input validation uses +cooperative super; metadata composition stays in the ABC with a small +per-input provenance hook, not a direct jump around an ancestor algorithm. + +The masked 4D control found that stacking shared 2D masks produces a 3D mask +outside the outer runtime axis's domain. The existing mask owner now consults +the composed ImageMaskDomain and uses its declared broadcast authority only +when required. Pixel values and exact masks are checked; no relaxed domain +check or ndarray coercion is used. Intermediate failed attempts remain in +diagnostics alongside the working logs and JUnit receipts. + +A broader metadata/artifact-consumer shard had 44 PASS / 1 FAIL. The failure, +test_callable_abi_keeps_main_flow_artifacts_in_trailing_slots, declares only +two outputs although identify_primary_objects now has seven diagnostic image +slots in addition to the canonical slots. The exact same test fails on a +complete unchanged production source snapshot of main 0c0563e65, not merely +an executor substitution. No production ABI assertion or fixture was weakened. +This pre-existing fixture mismatch is owned by the runtime repair worker under +parent integration. Its fixture is now aligned by deriving diagnostic artifact +specs from PrimaryObjectDiagnosticPlanes.artifact_specs, the same authority +used by the production module declaration. No new role/name inventory or +production relaxation is introduced. The original failure receipts stay +retained, including the unrelated failed shard invocation with nonexistent +test paths; corrected consumer checks are reported separately. + +Affected nominal owners: aligned runtime image stacks and their named-output +bundle subtype require materialization; already-dense image metadata/masked +payloads and arrays keep their own domains. Object labels, aligned non-image +values, measurement tables/columnar rows, relationships, sparse labels, +projectable identities and spatial graphs retain their registered strategies. +PURE_3D's prohibition on non-image slice-aligned kwargs is preserved. FULL_STACK +is shared by all processing contracts; ordinary slice modes are unchanged. +This is source evidence, not a claim that every backend/input combination was +executed or that GPU/device coverage is complete. + +Disposable baseline source snapshot: owner H003g runtime repair worker, +/home/ts/.cache/agent-scratch/h003g-runtime-repair-baseline-20261001, 31 MiB, +purpose unchanged-main fixture control. It contains no scientific data and is +recoverable from Git; logs are retained here before removing that owned scratch. + +Remaining gate: original R1 environment disposition, parent integration +and scheduled fresh installed entrypoint acceptance with synthetic inputs. +This source checkpoint is not live readiness or biological success. H003g +stays FAILED and is not a validation dataset for this repair. + +Guard disposition at source checkpoint 63ad7c652: original CI-pinned R0 +debt_ratchet.py (agent-comms 3b03785f45df2ef5dc62ba6aed99294192ecbb01, +SHA256 e323c94d49c2b72d9524a5169f123e64b4a6e46a41035ca9fb4497e49b6ca562) +and its direct owner dependencies are byte-identical in the existing cached +package. It ran with the CI's Python 3.14 using existing local dependencies: +production openhcs root PASS, 18.75 seconds wall / 89428 KiB RSS, one CPU, +512 MiB / no swap / 60 seconds. Earlier interpreter/dependency import failures +remain retained. Changed-source counts have no positive structural deltas. + +The original scripts.check_refactor_r1 is NOT passing: the existing current +NRA environment fails to import RedundantTypeCheckDetector from the detector +package before scanning. The worktree's recorded dependency gitlinks are also +uninitialized, so full-context source materialization is not yet qualified. +No detector substitution, narrowed context, script modification, download or +dependency installation has been used to manufacture a passing result. Parent +integration must disposition/qualify the original R1 environment separately. + +Merged-main source gate (2026-10-01) +----------------------------------- + +Main 7b0ec3f5ab5a35a586d77c480fb7d5d6b1c85ba0 was merged normally, without +rebase, force push, runtime install or frozen trial modification. Merge commit +b770266428c5faac96069011c636fe93a1712acc is the original R0 head; the only +subsequent source changes add source tests (no production changes). + +Original R0 compared exactly against main 7b0ec3f5: openhcs, scripts and +benchmark roots all PASS, respectively 17.65 / 2.52 / 3.37 seconds wall and +88244 / 59028 / 57892 KiB RSS. No detector or threshold was modified. All +shards retained one CPU, 512 MiB / no swap and a 60-second bound. + +111 family controls PASS after this merge (3.33 seconds pytest / 4.10 seconds +wall, 322492 KiB RSS), including a direct full-stack identity test for object +labels, measurement tables/columnar rows, spatial graphs, aligned non-image +tokens and already-dense image/array carriers. 45 consumer controls PASS +(4.01 seconds pytest / 5.27 seconds wall, 372736 KiB RSS), including the +declaration-derived diagnostic ABI fixture correction. These 156 source cases +supersede the initial 33-case checkpoint, but are not fresh installed numeric +or native acceptance. + +The two MRO cases contain real independent TEST capabilities: one changes +pixels, the other adds validity masks. Both use cooperative super through the +actual AlignedImageStack composition owner, and both orderings check exact +pixels AND masks. This proves that extension seam, not a new production MI +family or a complete NRA equivalence certificate. + +Raw receipts, unsuccessful attempts and source/artifact SHA256 manifests are +retained in docs/refactor/receipts/cp-aligned-full-stack-342-20261001.tar.gz; +diagnostics/README.rst indexes their interpretation and replay commands. +The worker's 31 MiB disposable unchanged-main source snapshot was removed +after its processes completed and failure receipts were retained; it can be +recreated from the recorded Git commit. No H003g scratch, source, environment, +input, harness, logs or sealed data was changed. + +Next acceptance belongs to the parent after #338: reviewed source must be +installed into a separately isolated candidate, then the actual installed +user entrypoint must execute matched synthetic aligned primary and auxiliary +images, retain exact typed outputs/masks/provenance and satisfy the raw +callable's dense ABI. Neither source tests nor old frozen H003g are substitutes +for that gate. Original R1 remains explicitly unqualified as described above. + +Binding nominal review / opaque-kwarg correction +------------------------------------------------ + +The current NRA skill and authoritative refactor-audit ZIP main, pattern +README, implementation, membership, boundaries and surface-receipt references +were reread personally. IMPL-12/13 and BOUND-2 still identify the actual shared +composition/projection owners; no new copied procedure or mirror is needed. + +The review tightened two witnesses. A new CalibratedAlignedImageStack supplies +only a cooperative per-input calibration hook; the existing registered MRO +projection and real rescale executor consume it with no dispatcher/executor +edit. Exact pixels, runtime axis, spacing and original source identities pass. +A named bundle now includes real inner runtime-plane provenance and asserts +the exact outer SOURCE_BINDING aliases and contributor paths after selection. +The bundle's identity metadata hook is an alternative axis policy: it must not +clear the named outer alias by applying the runtime-stack contributor promotion. +Both use the ancestor's actual compose algorithm, not a leaf copy or direct +ancestor jump. The older bare-array bundle test alone did not prove this. + +Full-stack projection must not impose the slice-mode registration requirement +on opaque kwargs. The facade now asks the existing nominal registry's optional +for_nominal_value resolver and preserves an unregistered whole-stack value +unchanged. Registered image owners still materialize through their hooks; +slice-mode strategy_for_value remains strict. Four added actual executor +controls prove opaque kwarg identity for every ProcessingContract, alongside +the typed non-image/image identity control. This fixes a regression risk in +published 34203beb1, not a new scientific input requirement or fallback reader. + +Local corrected source: 117 family cases PASS, 3.36 seconds pytest / 4.15 +seconds wall, 322644 KiB RSS, under the existing one CPU / 512 MiB / no swap / +60-second shard. The original 34203beb1 source/installed receipts are retained +and are not relabelled as acceptance of this correction. Parent explicitly +dispositioned that prepared predecessor before publication/rebuild. Original +R1 remains unqualified; updated exact-production R0 is recorded separately. diff --git a/docs/refactor/examples/344-aligned-rescale-engineering.py b/docs/refactor/examples/344-aligned-rescale-engineering.py new file mode 100644 index 0000000000..b0a2c41066 --- /dev/null +++ b/docs/refactor/examples/344-aligned-rescale-engineering.py @@ -0,0 +1,57 @@ +# OpenHCS pipeline +# Corrected engineering recipe; predecessor failure remains in Git/receipts. +from pathlib import Path +from openhcs.constants import AllComponents, GroupBy, Microscope, VariableComponents +from openhcs.constants.input_source import InputSource +from openhcs.core.artifacts import ArtifactInputPlan, ArtifactOutputPlan, ImageArtifactType +from openhcs.core.config import LazyPathPlanningConfig, LazyProcessingConfig, LazyVFSConfig, MaterializationBackend, PipelineConfig +from openhcs.core.source_bindings import ComponentSelector, LazySourceBindingsConfig, LazyStepSourceBindingsConfig, MetadataExtractionRule, MetadataSource, NamedSourceBinding, SourceBindingMatchDimension, SourceBindingMatchField, SourceBindingMatchMethod, SourceBindingMatchPlan, SourceFilterClause, SourceFilterMatchType, SourceFilterSubject, SourceSelector +from openhcs.core.source_metadata import SourceVoxelSpacing, SourceVoxelSpacingUnit +from openhcs.core.steps.function_step import FunctionStep +from openhcs.processing.backends.cellprofiler.intensity import RescaleIntensityModule, rescale_intensity + +primary_binding, reference_binding = RescaleIntensityModule.declared_artifact_bindings(plan_type=ArtifactInputPlan, artifact_type=ImageArtifactType) +(output_binding,) = RescaleIntensityModule.declared_artifact_bindings(plan_type=ArtifactOutputPlan, artifact_type=ImageArtifactType) + +pipeline_config = PipelineConfig( + num_workers=1, + use_threading=False, + microscope=Microscope.IMAGEXPRESS, + materialization_results_path=Path("results"), + path_planning_config=LazyPathPlanningConfig( + global_output_folder=Path("/home/ts/wt/openhcs-344-engineering-20261001/output"), + output_dir_suffix="_openhcs", + ), + vfs_config=LazyVFSConfig(materialization_backend=MaterializationBackend.DISK), + source_bindings_config=LazySourceBindingsConfig( + source_filters=(SourceFilterClause(SourceFilterSubject.EXTENSION, SourceFilterMatchType.IS_TIF),), + metadata_rules=(MetadataExtractionRule( + MetadataSource.FILE_NAME, + r"(?PA01)_s0*(?P[1-9][0-9]*)_w(?P[1-9][0-9]*)_z0*(?P[1-9][0-9]*)_t0*(?P[1-9][0-9]*)\.tif$", + ),), + match_plan=SourceBindingMatchPlan( + method=SourceBindingMatchMethod.METADATA, + dimensions=tuple(SourceBindingMatchDimension(fields=tuple( + SourceBindingMatchField(alias=alias, metadata_field=field) + for alias in ("EngineeringCH1", "EngineeringCH2") + )) for field in ("well", "site", "z_index", "timepoint")), + ), + source_voxel_spacing=SourceVoxelSpacing((4.0, 0.5, 0.5), SourceVoxelSpacingUnit.MICROMETERS), + bindings=( + NamedSourceBinding(alias="EngineeringCH1", selector=SourceSelector(components=(ComponentSelector(AllComponents.CHANNEL, "1"), ComponentSelector(AllComponents.SITE, "1"))), component_identity=(ComponentSelector(AllComponents.CHANNEL, "1"),)), + NamedSourceBinding(alias="EngineeringCH2", selector=SourceSelector(components=(ComponentSelector(AllComponents.CHANNEL, "2"), ComponentSelector(AllComponents.SITE, "1"))), component_identity=(ComponentSelector(AllComponents.CHANNEL, "2"),)), + ), + ), +) +pipeline_steps = [ + FunctionStep( + name="Engineering aligned full-stack RescaleIntensity", + func=(rescale_intensity, { + primary_binding.require_parameter_name(): "EngineeringCH1", + reference_binding.require_parameter_name(): "EngineeringCH2", + output_binding.require_parameter_name(): "EngineeringRescaled", + }), + processing_config=LazyProcessingConfig(input_source=InputSource.PIPELINE_START, variable_components=[VariableComponents.Z_INDEX], group_by=GroupBy.NONE), + source_bindings=LazyStepSourceBindingsConfig(enabled=True), + ), +] diff --git a/docs/refactor/receipts/cp-aligned-full-stack-342-20261001.tar.gz b/docs/refactor/receipts/cp-aligned-full-stack-342-20261001.tar.gz new file mode 100644 index 0000000000..4627f0db5b Binary files /dev/null and b/docs/refactor/receipts/cp-aligned-full-stack-342-20261001.tar.gz differ diff --git a/docs/refactor/receipts/cp-aligned-full-stack-342-a14-supplement.tar.gz b/docs/refactor/receipts/cp-aligned-full-stack-342-a14-supplement.tar.gz new file mode 100644 index 0000000000..337f193632 Binary files /dev/null and b/docs/refactor/receipts/cp-aligned-full-stack-342-a14-supplement.tar.gz differ diff --git a/docs/refactor/receipts/cp-aligned-full-stack-342-acquisition.tar.gz b/docs/refactor/receipts/cp-aligned-full-stack-342-acquisition.tar.gz new file mode 100644 index 0000000000..ceb9fd61e3 Binary files /dev/null and b/docs/refactor/receipts/cp-aligned-full-stack-342-acquisition.tar.gz differ diff --git a/docs/refactor/receipts/cp-stack-installed-native-344-20261001.tar.gz b/docs/refactor/receipts/cp-stack-installed-native-344-20261001.tar.gz new file mode 100644 index 0000000000..483b0fb922 Binary files /dev/null and b/docs/refactor/receipts/cp-stack-installed-native-344-20261001.tar.gz differ diff --git a/openhcs/core/aligned_image_payload.py b/openhcs/core/aligned_image_payload.py index bf9e42710b..23e33482c5 100644 --- a/openhcs/core/aligned_image_payload.py +++ b/openhcs/core/aligned_image_payload.py @@ -32,6 +32,7 @@ ImagePayloadSliceProjector, ImagePayloadMetadataCarrier, ImagePayloadMetadataCompositionMode, + ImageMaskDomain, image_payload_data, image_payload_mask, image_payload_metadata, @@ -396,8 +397,6 @@ def resolve_source_spatial_value(self, value: Any) -> Any: return adapter.value_in_payload_domain(reference) - - def stack_image_payload_context( image_payloads: Sequence[Any], stack: RuntimeArrayData, @@ -413,6 +412,78 @@ def stack_image_payload_context( return metadata.payload_with(stack, _stack_image_payload_mask(payloads, stack)) +class ImagePayloadStackComposition(ABC): + """Compose pixels, masks and provenance on one declared leading axis.""" + + @property + @abstractmethod + def composition_payloads(self) -> tuple[Any, ...]: ... + + @property + @abstractmethod + def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: ... + + def composition_source_metadata(self) -> tuple[ImagePayloadMetadata, ...]: + return tuple( + self.composition_payload_metadata(image_payload_metadata(payload)) + for payload in self.composition_payloads + ) + + def composition_payload_metadata( + self, metadata: ImagePayloadMetadata + ) -> ImagePayloadMetadata: + """Preserve each input's declared provenance unless the axis owner projects it.""" + return metadata + + def compose(self) -> Any: + payloads = self.composition_payloads + composed = self.compose_unmasked( + tuple(image_payload_data(payload) for payload in payloads) + ) + metadata = ImagePayloadMetadata.compose( + payloads, + mode=self.composition_metadata_mode, + source_metadata=self.composition_source_metadata(), + ) + return metadata.payload_with(composed, self.compose_mask(composed, metadata)) + + def compose_unmasked( + self, payloads: tuple[RuntimeArrayData, ...] + ) -> RuntimeArrayData: + memory_type = detect_memory_type(payloads[0]) + return stack_runtime_slices( + payloads, memory_type, MemoryType(memory_type).device_id_of(payloads[0]) + ) + + def compose_mask(self, composed: Any, metadata: ImagePayloadMetadata) -> Any | None: + return _stack_image_payload_mask( + self.composition_payloads, + composed, + output_mask_domain=metadata.mask_domain(composed), + ) + + +@dataclass(slots=True) +class ImagePayloadStackContext(ImagePayloadStackComposition): + """Explicit dense stack inputs; the ancestor owns composition.""" + + payloads: Sequence[RuntimeArrayData] + metadata_mode: ImagePayloadMetadataCompositionMode + + def __post_init__(self) -> None: + self.payloads = tuple(self.payloads) + if not self.payloads: + raise ValueError("Cannot stack an empty image payload sequence.") + + @property + def composition_payloads(self) -> tuple[Any, ...]: + return tuple(self.payloads) + + @property + def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: + return self.metadata_mode + + def stack_image_payloads( image_payloads: Sequence[Any], *, @@ -420,16 +491,7 @@ def stack_image_payloads( ) -> Any: """Stack image payloads in their declared memory domain with full context.""" - payloads = tuple(image_payloads) - if not payloads: - raise ValueError("Cannot stack an empty image payload sequence.") - arrays = tuple(image_payload_data(payload) for payload in payloads) - memory_type = detect_memory_type(arrays[0]) - return stack_image_payload_context( - payloads, - stack_runtime_slices(arrays, memory_type, 0), - metadata_mode=metadata_mode, - ) + return ImagePayloadStackContext(image_payloads, metadata_mode).compose() def stack_image_payload_context_from_metadata( @@ -452,6 +514,8 @@ def stack_image_payload_context_from_metadata( def _stack_image_payload_mask( image_payloads: Sequence[Any], stack: RuntimeArrayData, + *, + output_mask_domain: ImageMaskDomain | None = None, ) -> RuntimeArrayData | None: masks = tuple(image_payload_mask(payload) for payload in image_payloads) if not any(mask is not None for mask in masks): @@ -475,6 +539,18 @@ def _stack_image_payload_mask( strict=True, ) ) + stacked_mask_shape = (len(payloads), *tuple(np.shape(resolved_masks[0]))) + if output_mask_domain is not None and not output_mask_domain.accepts( + stacked_mask_shape + ): + resolved_masks = tuple( + image_payload_metadata(payload) + .mask_domain(slice_domain) + .broadcast_to_data(mask) + for payload, slice_domain, mask in zip( + payloads, output_slice_domains, resolved_masks, strict=True + ) + ) return stack_runtime_slices( resolved_masks, detect_memory_type(stack), @@ -692,18 +768,15 @@ def preserved_plane_projection( @dataclass(slots=True) -class ImagePayloadBundleContext: +class ImagePayloadBundleContext(ImagePayloadStackContext): """Compose same-slice image bundle data, masks, and metadata together.""" - payloads: Sequence[RuntimeArrayData] metadata_mode: ImagePayloadMetadataCompositionMode = ( ImagePayloadMetadataCompositionMode.BUNDLE ) def __post_init__(self) -> None: - self.payloads = tuple(self.payloads) - if not self.payloads: - raise ValueError("ImagePayloadBundleContext.payloads cannot be empty.") + super(ImagePayloadBundleContext, self).__post_init__() declared_axes = tuple( ( index, @@ -724,7 +797,7 @@ def __post_init__(self) -> None: @property def source_metadata(self) -> tuple[ImagePayloadMetadata, ...]: - return tuple(image_payload_metadata(payload) for payload in self.payloads) + return self.composition_source_metadata() @property def data_payloads(self) -> tuple[RuntimeArrayData, ...]: @@ -754,17 +827,6 @@ def from_payloads( metadata_mode=metadata_mode, ) - def compose(self) -> Any: - composed = self.compose_unmasked(self.data_payloads) - metadata = ImagePayloadMetadata.compose( - self.payloads, - mode=self.metadata_mode, - ) - return metadata.payload_with( - composed, - self.compose_mask(composed, metadata), - ) - def compose_mask( self, composed: Any, @@ -1064,12 +1126,29 @@ def __post_init__(self) -> None: @dataclass(slots=True) -class AlignedImageStack: +class AlignedImageStack(ImagePayloadStackComposition): """Per-slice multi-image bundles aligned to one OpenHCS stack.""" slices: tuple[Any, ...] slice_contexts: tuple[AlignedImageSliceContext, ...] = () + @property + def composition_payloads(self) -> tuple[Any, ...]: + return self.slices + + @property + def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: + return ImagePayloadMetadataCompositionMode.STACK + + def composition_payload_metadata( + self, metadata: ImagePayloadMetadata + ) -> ImagePayloadMetadata: + """Inner image bundles contribute provenance, not an outer slice axis.""" + metadata = super(AlignedImageStack, self).composition_payload_metadata(metadata) + return metadata.with_source_provenance( + metadata.source_provenance.with_runtime_planes_as_contributors() + ) + def __post_init__(self) -> None: self.slices = tuple(self.slices) self.slice_contexts = tuple(self.slice_contexts) @@ -1144,8 +1223,25 @@ def output_payload( class ImageOutputBundle(AlignedImageStack): """Named main-flow image outputs sharing one invocation context.""" + @property + def composition_payloads(self) -> tuple[Any, ...]: + return tuple( + context.contextualize_image_payload(payload) + for payload, context in zip(self.slices, self.slice_contexts, strict=True) + ) + + @property + def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: + return ImagePayloadMetadataCompositionMode.BUNDLE + + def composition_payload_metadata( + self, metadata: ImagePayloadMetadata + ) -> ImagePayloadMetadata: + """Named output surfaces remain source-binding planes, not runtime slices.""" + return metadata + def __post_init__(self) -> None: - AlignedImageStack.__post_init__(self) + super(ImageOutputBundle, self).__post_init__() if not self.slice_contexts or any( context.is_anonymous_main_flow for context in self.slice_contexts ): diff --git a/openhcs/core/runtime_slice_projection.py b/openhcs/core/runtime_slice_projection.py index dbc82ace20..283b9a9e84 100644 --- a/openhcs/core/runtime_slice_projection.py +++ b/openhcs/core/runtime_slice_projection.py @@ -421,6 +421,10 @@ def identity_projected_value( del context return value + def full_stack_value(self, value: RuntimeProjectionData) -> RuntimeProjectionData: + """Preserve values already in their callable's whole-stack domain.""" + return value + def slice_count_for_value( self, value: RuntimeProjectionData, @@ -506,6 +510,9 @@ class AlignedImageStackRuntimeSliceProjectionStrategy(RuntimeSliceProjectionStra value_type = AlignedImageStack + def full_stack_value(self, value: RuntimeProjectionData) -> RuntimeProjectionData: + return cast(AlignedImageStack, value).compose() + def value_for_slice( self, value: RuntimeProjectionData, @@ -539,7 +546,9 @@ def slice_count_for_value( return len(cast(AlignedImageStack, value).slices) -class ImageOutputBundleRuntimeSliceProjectionStrategy(RuntimeSliceProjectionStrategy): +class ImageOutputBundleRuntimeSliceProjectionStrategy( + AlignedImageStackRuntimeSliceProjectionStrategy +): """Project each named output through its shared declared runtime axis.""" value_type = ImageOutputBundle @@ -881,6 +890,18 @@ def slice_count_for_value( class RuntimeSliceProjection: """SSOT for runtime-slice count and value projection.""" + @classmethod + def full_stack_value(cls, value: RuntimeProjectionData) -> RuntimeProjectionData: + """Materialize declared alignment; preserve opaque whole-stack arguments.""" + strategy = RuntimeSliceProjectionStrategy.for_nominal_value(value) + return value if strategy is None else strategy.full_stack_value(value) + + @classmethod + def full_stack_kwargs( + cls, kwargs: Mapping[str, RuntimeProjectionData] + ) -> dict[str, RuntimeProjectionData]: + return {name: cls.full_stack_value(value) for name, value in kwargs.items()} + @classmethod def preserved_context_for_value( cls, diff --git a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py index d6667d5f76..c1279ed9cd 100644 --- a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py +++ b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py @@ -305,8 +305,8 @@ def execute_pure_3d( ) -> RuntimeCallableArgument: function_name = callable_contract.function_name projection_started_at = time.perf_counter() - projected_image = image - projected_kwargs = dict(kwargs) + projected_image = RuntimeSliceProjection.full_stack_value(image) + projected_kwargs = RuntimeSliceProjection.full_stack_kwargs(kwargs) if callable_contract.processing_contract is ProcessingContract.PURE_3D: _validate_pure_3d_kwargs_do_not_carry_runtime_slice_alignment( callable_contract, diff --git a/tests/unit/test_cellprofiler_full_stack_materialization.py b/tests/unit/test_cellprofiler_full_stack_materialization.py new file mode 100644 index 0000000000..d4fd62b96b --- /dev/null +++ b/tests/unit/test_cellprofiler_full_stack_materialization.py @@ -0,0 +1,399 @@ +"""Dense full-stack ABI must consume aligned source/artifact values correctly.""" + +import numpy as np +import pytest + +from openhcs.core.aligned_image_payload import ( + AlignedImageSliceContext, + AlignedImageStack, + ImageOutputBundle, + ImagePayloadExecutionMode, + ImagePayloadStackComposition, + compose_aligned_image_payload, +) +from openhcs.core.callable_contract import CallableContract, CallableMetadata +from openhcs.core.measurement_row_materialization import MeasurementSparseColumnarRows +from openhcs.core.runtime_measurements import ( + MeasurementScope, + MeasurementSubject, + MeasurementTable, +) +from openhcs.core.runtime_object_labels import ObjectLabelPayload, ObjectLabelVariantData +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, +) +from openhcs.core.runtime_plane_projection import ( + RuntimePlaneAxis, + RuntimePlaneAxisValueProjection, +) +from openhcs.core.runtime_slice_alignment import RuntimeSliceAlignedValues +from openhcs.core.runtime_slice_projection import RuntimeSliceProjection +from openhcs.core.runtime_spatial_graph import SpatialGraph, SpatialGraphNode +from openhcs.core.runtime_tabular_values import FieldSpec +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import SourceVoxelSpacing, SourceVoxelSpacingUnit +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( + CellProfilerFunctionContractExecutor, +) +from openhcs.processing.backends.cellprofiler.intensity import rescale_intensity +from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract + + +def test_real_rescale_full_stack_materializes_composed_runtime_sources(): + data = np.arange(24, dtype=np.float32).reshape(2, 3, 4) + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE + ).payload_with(data, None) + composition = compose_aligned_image_payload( + "synthetic source inputs", (source, source) + ) + assert isinstance(composition.payload, AlignedImageStack) + contract = CallableContract.from_callable(rescale_intensity) + result = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + composition.payload, + {}, + execution_mode=contract.runtime_image_execution_mode, + ) + expected = np.stack((data, data), axis=1) / 23.0 + np.testing.assert_allclose(image_payload_data(result), expected) + + +@pytest.mark.parametrize("processing_contract", tuple(ProcessingContract)) +@pytest.mark.parametrize("slice_count", (1, 2)) +def test_every_full_stack_processing_family_materializes_image_and_image_kwargs( + processing_contract, slice_count +): + planes = tuple( + np.full((3, 4), index, dtype=np.float32) for index in range(slice_count) + ) + aligned = AlignedImageStack(planes) + opaque = RuntimeSliceAlignedValues(("not an image",)) + calls = [] + + def dense_callable(image, *, reference, token): + calls.append((image, reference, token)) + assert not isinstance(image, AlignedImageStack) + assert not isinstance(reference, AlignedImageStack) + assert token is opaque + np.testing.assert_array_equal(image_payload_data(image), np.stack(planes)) + np.testing.assert_array_equal(image_payload_data(reference), np.stack(planes)) + return image + + contract = CallableContract( + func=dense_callable, + function_name="dense_callable", + module_name="DenseProbeModule", + metadata=CallableMetadata(processing_contract=processing_contract), + ) + kwargs = {"reference": aligned, "token": opaque} + # Retain the PURE_3D prohibition on slice-aligned non-image kwargs. + if processing_contract is ProcessingContract.PURE_3D: + with pytest.raises(ValueError, match="runtime-slice-aligned kwargs.*token"): + CellProfilerFunctionContractExecutor().execute( + contract, + dense_callable, + aligned, + kwargs, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + assert calls == [] + kwargs["token"] = None + opaque = None + result = CellProfilerFunctionContractExecutor().execute( + contract, + dense_callable, + aligned, + kwargs, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + assert len(calls) == 1 + np.testing.assert_array_equal(image_payload_data(result), np.stack(planes)) + assert kwargs["reference"] is aligned + + +def test_aligned_materialization_retains_masks_calibration_and_exact_contributors(): + spacing = SourceVoxelSpacing((1.25, 2.5, 3.0), SourceVoxelSpacingUnit.MICROMETERS) + sources = [] + expected_paths = [] + for channel, alias in ((1, "DNA"), (2, "actin")): + paths = tuple(f"/synthetic/A01_s{site}_w{channel}.tif" for site in (1, 2)) + expected_paths.extend(paths) + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_names=(alias,), + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=paths, + component_metadata=tuple( + {"well": "A01", "site": str(site), "channel": str(channel)} + for site in (1, 2) + ), + ), + source_voxel_spacing=spacing, + source_dtype="uint8", + intensity_scale=255.0, + source_spatial_domain=SourceSpatialDomain( + source_shape_yx=(3, 4), origin_yx=(0, 0) + ), + ) + pixels = np.full((2, 3, 4), channel, dtype=np.float32) + mask = np.ones_like(pixels, dtype=bool) + mask[0, 0, 0] = False + sources.append(metadata.payload_with(pixels, mask)) + composed = compose_aligned_image_payload( + "matched synthetic inputs", tuple(sources) + ).payload + dense = RuntimeSliceProjection.full_stack_value(composed) + assert dense.shape == (2, 2, 3, 4) + metadata = image_payload_metadata(dense) + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_voxel_spacing == spacing + assert metadata.source_spatial_shape_yx == (3, 4) + assert set(metadata.source_provenance.represented_source_image_names) == { + "DNA", + "actin", + } + assert { + identity.path + for identity in metadata.source_provenance.represented_source_identities + } == set(expected_paths) + np.testing.assert_array_equal( + image_payload_mask(dense), + np.stack( + [ + np.broadcast_to(image_payload_mask(plane), (2, 3, 4)) + for plane in composed.slices + ] + ), + ) + for index in range(2): + projected = RuntimeSliceProjection.value_for_slice( + dense, + RuntimePlaneAxisValueProjection.from_selected_plane( + axis=RuntimePlaneAxis.RUNTIME_SLICE, plane_index=index, axis_size=2 + ), + ) + np.testing.assert_array_equal( + image_payload_data(projected), image_payload_data(composed.slices[index]) + ) + expected = {f"/synthetic/A01_s{index + 1}_w{channel}.tif" for channel in (1, 2)} + assert { + identity.path + for identity in image_payload_metadata( + projected + ).source_provenance.represented_source_identities + } == expected + + +def test_named_output_bundle_materializes_its_own_source_binding_axis(): + planes = tuple(np.full((3, 4), index, dtype=np.float32) for index in (1, 2)) + bundle = ImageOutputBundle( + planes, + tuple( + AlignedImageSliceContext.main_flow(name, artifact_kind="image") + for name in ("red", "green") + ), + ) + dense = RuntimeSliceProjection.full_stack_value(bundle) + assert image_payload_metadata(dense).plane_axis is RuntimePlaneAxis.SOURCE_BINDING + assert image_payload_metadata( + dense + ).source_provenance.represented_source_image_names == ("red", "green") + np.testing.assert_array_equal(image_payload_data(dense), np.stack(planes)) + assert bundle.slices == planes + + +def test_named_bundle_preserves_outer_aliases_with_real_inner_plane_provenance(): + spacing = SourceVoxelSpacing((1.25, 2.5, 3.0), SourceVoxelSpacingUnit.MICROMETERS) + payloads = [] + for channel, alias in enumerate(("red", "green"), start=1): + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_names=(alias,), + source_voxel_spacing=spacing, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=tuple(f"/synthetic/{alias}_s{site}.tif" for site in (1, 2)), + component_metadata=tuple( + {"site": str(site), "channel": str(channel)} for site in (1, 2) + ), + ), + ) + payloads.append(metadata.payload_with(np.full((2, 3, 4), channel), None)) + bundle = ImageOutputBundle( + tuple(payloads), + tuple(AlignedImageSliceContext.main_flow(alias) for alias in ("red", "green")), + ) + dense = RuntimeSliceProjection.full_stack_value(bundle) + metadata = image_payload_metadata(dense) + assert metadata.plane_axis is RuntimePlaneAxis.SOURCE_BINDING + assert metadata.source_image_names == ("red", "green") + assert metadata.source_provenance.source_plane_count == 2 + assert metadata.source_voxel_spacing == spacing + for index, alias in enumerate(("red", "green")): + selected = RuntimeSliceProjection.value_for_slice( + dense, + RuntimePlaneAxisValueProjection.from_selected_plane( + axis=RuntimePlaneAxis.SOURCE_BINDING, plane_index=index, axis_size=2 + ), + ) + np.testing.assert_array_equal( + image_payload_data(selected), image_payload_data(payloads[index]) + ) + selected_metadata = image_payload_metadata(selected) + assert selected_metadata.source_image_names == (alias,) + assert { + identity.path + for identity in selected_metadata.source_provenance.represented_source_identities + } == {f"/synthetic/{alias}_s{site}.tif" for site in (1, 2)} + + +def test_new_calibrated_nominal_stack_reaches_real_callable_without_dispatch_edits(): + spacing = SourceVoxelSpacing((1.25, 2.5, 3.0), SourceVoxelSpacingUnit.MICROMETERS) + + class CalibratedAlignedImageStack(AlignedImageStack): + """A generated acquisition grid owns one additional metadata fact.""" + + def composition_payload_metadata(self, metadata): + return super().composition_payload_metadata(metadata).replace_fields( + source_voxel_spacing=spacing + ) + + data = np.arange(24, dtype=np.float32).reshape(2, 3, 4) + payloads = tuple( + ImagePayloadMetadata(source_path=f"/synthetic/site_{index}.tif").payload_with( + plane, None + ) + for index, plane in enumerate(data) + ) + aligned = CalibratedAlignedImageStack(payloads) + contract = CallableContract.from_callable(rescale_intensity) + result = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + aligned, + {}, + execution_mode=contract.runtime_image_execution_mode, + ) + np.testing.assert_allclose(image_payload_data(result), data / 23.0) + metadata = image_payload_metadata(result) + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_voxel_spacing == spacing + assert { + identity.path + for identity in metadata.source_provenance.represented_source_identities + } == {"/synthetic/site_0.tif", "/synthetic/site_1.tif"} + + +def test_ragged_aligned_images_fail_before_dense_callable(): + aligned = AlignedImageStack((np.zeros((2, 3)), np.zeros((3, 3)))) + with pytest.raises(ValueError, match="one exact shape"): + RuntimeSliceProjection.full_stack_value(aligned) + + +def test_full_stack_preserves_nonimage_identity_domains_and_dense_images(): + rows = MeasurementSparseColumnarRows.from_rows( + ({"object_label": 1, "value": 2.5},), + fields=(FieldSpec("object_label", int), FieldSpec("value", float)), + ) + table = MeasurementTable( + name="Measurements", + rows=rows, + subject=MeasurementSubject(MeasurementScope.OBJECT, name="Objects"), + ) + labels = ObjectLabelPayload( + variant_data=ObjectLabelVariantData(labels=np.zeros((2, 3), dtype=np.int32)), + ) + graph = SpatialGraph( + name="graph", + nodes=(SpatialGraphNode(1, (2.0, 3.0)),), + edges=(), + ) + array = np.ones((2, 3), dtype=np.float32) + image = ImagePayloadMetadata(source_path="/synthetic/source.tif").payload_with( + array, array > 0 + ) + aligned_tokens = RuntimeSliceAlignedValues((labels, table)) + opaque = object() + kwargs = { + "rows": rows, + "table": table, + "labels": labels, + "graph": graph, + "tokens": aligned_tokens, + "array": array, + "image": image, + "opaque": opaque, + } + materialized = RuntimeSliceProjection.full_stack_kwargs(kwargs) + assert materialized is not kwargs + for name, value in kwargs.items(): + assert materialized[name] is value + + +@pytest.mark.parametrize("processing_contract", tuple(ProcessingContract)) +def test_full_stack_raw_callable_keeps_opaque_nonimage_kwargs(processing_contract): + opaque = object() + image = np.ones((2, 3, 4), dtype=np.float32) + calls = [] + + def consume(image, *, options): + calls.append(options) + assert options is opaque + return image + + contract = CallableContract( + func=consume, + function_name="consume", + module_name="OpaqueFullStackProbe", + metadata=CallableMetadata(processing_contract=processing_contract), + ) + result = CellProfilerFunctionContractExecutor().execute( + contract, + consume, + image, + {"options": opaque}, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + assert calls == [opaque] + assert image_payload_data(result) is image + + +@pytest.mark.parametrize("reverse", (False, True)) +def test_independent_composition_capabilities_cooperate_in_both_mro_orders(reverse): + calls = [] + + class PixelCapability: + def compose_unmasked(self, payloads): + calls.append("pixels") + return super().compose_unmasked(payloads) + 1 + + class MaskCapability: + def compose_mask(self, composed, metadata): + calls.append("mask") + mask = super().compose_mask(composed, metadata) + if mask is None: + mask = np.ones_like(composed, dtype=bool) + mask[:, 0, 0] = False + return mask + + bases = ( + (MaskCapability, PixelCapability) + if reverse + else (PixelCapability, MaskCapability) + ) + probe_type = type("IndependentCompositionProbe", (*bases, AlignedImageStack), {}) + probe = probe_type((np.zeros((2, 3)), np.ones((2, 3)))) + assert isinstance(probe, ImagePayloadStackComposition) + dense = RuntimeSliceProjection.full_stack_value(probe) + assert calls == ["pixels", "mask"] + np.testing.assert_array_equal( + image_payload_data(dense), np.stack((np.ones((2, 3)), np.full((2, 3), 2))) + ) + expected_mask = np.ones((2, 2, 3), dtype=bool) + expected_mask[:, 0, 0] = False + np.testing.assert_array_equal(image_payload_mask(dense), expected_mask) diff --git a/tests/unit/test_cellprofiler_runtime_input_binding_contract.py b/tests/unit/test_cellprofiler_runtime_input_binding_contract.py index 4902047a6f..12b438abf9 100644 --- a/tests/unit/test_cellprofiler_runtime_input_binding_contract.py +++ b/tests/unit/test_cellprofiler_runtime_input_binding_contract.py @@ -40,6 +40,9 @@ RuntimeInputBindingRequest, ) from openhcs.processing.backends.cellprofiler import identify_primary_objects +from openhcs.processing.backends.cellprofiler.primary_object_diagnostics import ( + PrimaryObjectDiagnosticPlanes, +) from openhcs.processing.backends.cellprofiler.watershed import watershed_cellprofiler4 from tests.unit.cellprofiler_runtime_test_support import ( @@ -653,7 +656,7 @@ def process(image: np.ndarray) -> np.ndarray: CellProfilerModuleCallableABI.validate_callable_artifact_abi(process, contract) -def test_callable_abi_keeps_main_flow_artifacts_in_trailing_slots() -> None: +def test_callable_abi_keeps_main_flow_artifacts_in_declared_slots() -> None: source = ArtifactSpec.input("Input", ImageArtifactType) measurements = ArtifactSpec.output("Measurements", MeasurementsArtifactType) labels = ArtifactSpec.output( @@ -662,13 +665,21 @@ def test_callable_abi_keeps_main_flow_artifacts_in_trailing_slots() -> None: relations=(GroupLineageSourceRelation(source=source.ref()),), ) - for func in (identify_primary_objects, watershed_cellprofiler4): + for func, diagnostics in ( + ( + identify_primary_objects, + PrimaryObjectDiagnosticPlanes.artifact_specs( + source_image=source, objects=labels + ), + ), + (watershed_cellprofiler4, ()), + ): contract = _contract(func, (source,)) contract = replace( contract, metadata=replace( contract.metadata, - artifact_outputs=(measurements, labels), + artifact_outputs=(measurements, labels, *diagnostics), ), )