diff --git a/docs/audits/architecture_a_m.json b/docs/audits/architecture_a_m.json index 2de6bf56fc..a4e273211a 100644 --- a/docs/audits/architecture_a_m.json +++ b/docs/audits/architecture_a_m.json @@ -168,7 +168,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares AnalysisConsolidationConfig and its standalone consolidation policy." }, { @@ -178,7 +178,7 @@ }, { "path": "tests/unit/test_function_artifact_materialization.py", - "sha256": "4ae063cac0f93a03b5b4aab7304c8d7e6bcfc04ec74d107ea14c94f76be755ca", + "sha256": "ba4f8f5f165ab87c3000d6d371a6f25cf4ec6a5d39b5047abaa392c827fe9ae8", "role": "Proves consolidation consumes runtime CSV values, uses the compiled backend destination, ignores non-table artifacts, and propagates failures." }, { @@ -188,7 +188,7 @@ }, { "path": "openhcs/processing/materialization/core.py", - "sha256": "1372625f3d7dc04cf6822a1887871dfa9a149b2313df7d7acefd452a6ad0f932", + "sha256": "1f5a89a1eb50766ef093df3fece16d6bb03422b92d062fe8ae6471656b1ebd45", "role": "Owns writer runtime-path selection before rendering and the shared ROI target request." } ], @@ -675,7 +675,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares OpenHCS domain configuration that consumes ObjectState mechanics." } ], @@ -841,7 +841,7 @@ }, { "path": "openhcs/core/context/processing_context.py", - "sha256": "9384fcc4d5ed7ea8607257ec448958f85a88a5caa6ae06f22142cc51b58b479e", + "sha256": "c7b79f50661c528261ce8c577fd67c753c84a5e499aa87cd5e540b53417351a2", "role": "Owns the compiled multiprocessing-axis identity for one processing context." }, { @@ -912,7 +912,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the OpenHCS global, pipeline, step, and standalone configuration hierarchy." } ], @@ -1142,7 +1142,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares standalone ExperimentalAnalysisConfig options and all directory-workflow filenames." }, { @@ -1198,7 +1198,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Pins the eight first-party package constraints and documents their installed integration surface." }, { @@ -1805,7 +1805,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the WellFilterConfig family preview policy and WellFilterMode member operators." }, { @@ -1955,7 +1955,7 @@ }, { "path": "tests/unit/test_config_document.py", - "sha256": "c6b291080398669b8449cb6ddaa78350f6c5b09f4f0cc57ebe92ac5f32460915", + "sha256": "0ee12db4633ab39b34f630e4ee1b0c48b40d8226f91fddc504bb3311b9612cda", "role": "Proves explicit, defaulted, and invalid typed field projections from canonical configuration source." }, { diff --git a/docs/audits/architecture_n_z_development.json b/docs/audits/architecture_n_z_development.json index 55003c0404..78b0415766 100644 --- a/docs/audits/architecture_n_z_development.json +++ b/docs/audits/architecture_n_z_development.json @@ -220,7 +220,7 @@ }, { "path": "openhcs/core/orchestrator/orchestrator.py", - "sha256": "ca733be328ed5f1f033ee521fbf9bf0b088791bd19542f0071ddb01b4f2d3d9c", + "sha256": "843e994d7b16592f43a15ce36c33a51ead534cf22df792838ce2b800621114b7", "role": "PipelineOrchestrator configuration ownership, resolution, compiler setup, and executor lifecycle." }, { @@ -250,7 +250,7 @@ }, { "path": "openhcs/core/context/processing_context.py", - "sha256": "9384fcc4d5ed7ea8607257ec448958f85a88a5caa6ae06f22142cc51b58b479e", + "sha256": "c7b79f50661c528261ce8c577fd67c753c84a5e499aa87cd5e540b53417351a2", "role": "Owns explicit release of the context-local image-stack cache without discarding observations." }, { @@ -290,7 +290,7 @@ "authority": [ { "path": "openhcs/core/orchestrator/orchestrator.py", - "sha256": "ca733be328ed5f1f033ee521fbf9bf0b088791bd19542f0071ddb01b4f2d3d9c", + "sha256": "843e994d7b16592f43a15ce36c33a51ead534cf22df792838ce2b800621114b7", "role": "PipelineOrchestrator configuration ownership, resolution, compiler setup, and executor lifecycle." }, { @@ -512,7 +512,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Pipeline, global, step, processing, and source configuration declarations." }, { @@ -572,7 +572,7 @@ }, { "path": "openhcs/core/context/processing_context.py", - "sha256": "9384fcc4d5ed7ea8607257ec448958f85a88a5caa6ae06f22142cc51b58b479e", + "sha256": "c7b79f50661c528261ce8c577fd67c753c84a5e499aa87cd5e540b53417351a2", "role": "Owns the compiled multiprocessing-axis identity carried by each runtime context." }, { @@ -822,7 +822,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Pipeline, global, step, processing, and source configuration declarations." }, { @@ -1020,7 +1020,7 @@ }, { "path": "openhcs/processing/materialization/core.py", - "sha256": "1372625f3d7dc04cf6822a1887871dfa9a149b2313df7d7acefd452a6ad0f932", + "sha256": "1f5a89a1eb50766ef093df3fece16d6bb03422b92d062fe8ae6471656b1ebd45", "role": "Nominal artifact materialisation and ROI projection strategies." }, { @@ -1408,7 +1408,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package discovery, and entry-point authority." } ], @@ -1646,7 +1646,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package discovery, and entry-point authority." }, { @@ -1719,7 +1719,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package-discovery, and installed entry-point authority." }, { @@ -2612,7 +2612,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns nominal gallery scenario identity, publication targets, representative images, copy, accessibility text, dimensions, contextual and stable-window capture targets, and derivative contracts." }, { @@ -2659,7 +2659,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package discovery, and entry-point authority." }, { @@ -2763,7 +2763,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package discovery, and entry-point authority." }, { @@ -2811,7 +2811,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package discovery, and entry-point authority." }, { @@ -2869,7 +2869,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the nominal gallery scenario lattice, stable-window and contextual capture targets, publication targets, derivative inventory, source evidence, card rendering, and generated release record." }, { @@ -3521,7 +3521,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package discovery, and entry-point authority." }, { @@ -3712,7 +3712,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Current Python support, dependency, package discovery, and entry-point authority." } ], diff --git a/docs/audits/pr129_tiff_compression_review_20260924.md b/docs/audits/pr129_tiff_compression_review_20260924.md new file mode 100644 index 0000000000..a38b161206 --- /dev/null +++ b/docs/audits/pr129_tiff_compression_review_20260924.md @@ -0,0 +1,55 @@ +# PR #129 TIFF compression authority-delta review + +Date: 2026-09-24 + +The opt-in disk-TIFF writer change invalidated 34 recorded authority edges +across 29 active documentation pages. I re-read each complete dependent page +against the changed OpenHCS declarations, runtime path, and tests, and the +PolyStore codec declaration. The generated configuration reference remains the +authority for exact fields, accepted values, and defaults; it is not duplicated +as a hand-maintained field table. + +Two pages needed prose changes. `concepts/storage_system.rst` now explains the +separation of disk TIFF compression from Zarr compression and semantic +materialization. `guide_for_biologists/configuration_reference.rst` now names +the `TiffConfig` family and its scope in the practitioner mental model. Both +remain explanations rather than becoming exhaustive field references. The +other 27 pages retain their existing claims and Diátaxis purposes: the new +config is resolved from the existing pipeline declaration, carried in the +compiled processing context, and used only on disk TIFF writes, so their +descriptions of inheritance, runtime ownership, ROI, MCP schema projection, +analysis, and viewer behavior remain accurate. + +This is an editorial receipt, not product authority. The authoritative files +are the declarations, implementations, and tests recorded in the JSON audit +entries. The changed page and authority digests were refreshed only after this +comparison. Documentation validation passed for 155 audited sources, the full +Sphinx HTML build succeeded with warnings treated as errors, and 15 focused +documentation, MCP, and TIFF-output tests passed. At this initial review, +the separate published-dependency readiness gate remained open because the +PolyStore change had not yet been released. + +The first CI pass found three Black-only formatting differences. After running +the pinned formatter, the changed materialization owner files differ only in +line wrapping. I rechecked the complete analysis-consolidation, ROI, and storage +pages against those unchanged semantics before refreshing their three affected +authority digests. The reformatted test file is not a recorded audit authority. + +## Published-dependency follow-up + +PolyStore PR #11 was merged and released as v0.2.19. Its publish workflow +passed, and both the wheel and source distribution are present on PyPI. +OpenHCS now requires `polystore>=0.2.19,<0.3` and pins the PolyStore +submodule to release commit `0efe67fdd14985bf90cee6e0f0c4735d41d265d1`. +The 17 documentation audit entries that depend on `pyproject.toml` were +rechecked against the new dependency floor and their authority digests +refreshed. No affected explanatory page asserted the prior floor. + +The subsequent website CI run exposed an eager import in the gallery catalog: +its source-capture request used `ExecutionConnectionSpec` only as a postponed +type annotation, but importing that DTO loaded the scientific runtime into +the deliberately minimal website environment. The import is now restricted to +type checking. The gallery's runtime declarations, capture behavior, and +published projections are unchanged. The 13 audited pages referencing the +gallery catalog were checked for claims about those surfaces, and their +authority digests were refreshed. diff --git a/docs/audits/project_surface.json b/docs/audits/project_surface.json index 5f0ca69728..ab29a11a7f 100644 --- a/docs/audits/project_surface.json +++ b/docs/audits/project_surface.json @@ -12,7 +12,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Declares the published README, Python range, package dependencies, optional surfaces, console commands, and project links." }, { diff --git a/docs/audits/user_docs.json b/docs/audits/user_docs.json index cf587ac4db..79b610ca7a 100644 --- a/docs/audits/user_docs.json +++ b/docs/audits/user_docs.json @@ -11,7 +11,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -21,7 +21,7 @@ }, { "path": "openhcs/core/orchestrator/orchestrator.py", - "sha256": "ca733be328ed5f1f033ee521fbf9bf0b088791bd19542f0071ddb01b4f2d3d9c", + "sha256": "843e994d7b16592f43a15ce36c33a51ead534cf22df792838ce2b800621114b7", "role": "Owns execution orchestration, worker submission, progress handling, and cleanup of compiled bundles." }, { @@ -171,7 +171,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Declares the package version, Python requirement, dependency extras, scripts, and distribution metadata." }, { @@ -495,7 +495,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -674,7 +674,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -707,7 +707,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Declares the package version, Python requirement, dependency extras, scripts, and distribution metadata." }, { @@ -727,7 +727,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -843,7 +843,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -871,7 +871,7 @@ }, { "path": "docs/source/concepts/storage_system.rst", - "source_sha256": "fa377aa5280583609ca43d6190cc03aaabb1a7abed40dd4344d8e686866ce1f7", + "source_sha256": "6457c0c132bd45f195274e51b255f45713b5efda9ee7ada352de556ac4aa1dde", "audience": [ "pipeline authors", "storage integrators" @@ -881,7 +881,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -891,7 +891,7 @@ }, { "path": "openhcs/core/steps/function_artifact_materialization.py", - "sha256": "5a219f74d1abfd59f915aae06cb3dc2befa1d1a83b82b59d6a8f98f3ac5865ba", + "sha256": "18c809a2d511501c4e308f6e92c18d5bb59744d4d3f59bb2bfc2507f2a2bdd03", "role": "Owns the rules that materialise function outputs through the configured storage backend." }, { @@ -899,14 +899,37 @@ "sha256": "ae641affae21b61448816a14bd68b7f2f3fc0082f0f1fbe51079ed32520450b1", "role": "Declares the storage backend interface and lifecycle shared by local and remote materialisation backends." }, + { + "path": "external/PolyStore/src/polystore/config.py", + "sha256": "62c8acad3da3abed8ccf7f1166fc6a0d094be1c3e842cec6446780541df292a0", + "role": "Declares lossless disk-TIFF codec settings, defaults, and path-specific batching." + }, + { + "path": "external/PolyStore/src/polystore/disk.py", + "sha256": "cf089b8c79f86099a7cea1b6cbe43ceaec6331196a44b43183f2e64b7746c219", + "role": "Applies TIFF codec settings in the disk writer without changing other formats." + }, + { + "path": "external/arraybridge/src/arraybridge/decorators.py", + "sha256": "99823105b0ae59054df1d84013633b8b6628d0a6dc3326383199ac7c65c69bed", + "role": "Applies declared dtype conversion to a callable result before materialisation can persist it." + }, { "path": "tests/unit/test_runtime_value_store.py", "sha256": "8900a7b83b59d0a40f9698dbafaaba778df982f9d7a0b628f80eab97335aeab0", "role": "Proves scoped typed runtime-value insertion, lookup, isolation, and cleanup." + }, + { + "path": "tests/unit/test_tiff_output_compression.py", + "sha256": "458f576e545a033ed9e23b9dc349ab92c19d296ec7f0e269ea89884cd9b82942", + "role": "Proves configured main-flow and named-artifact disk-TIFF writes round-trip exactly." } ], - "findings": [], - "disposition": "keep", + "findings": [ + "The storage explanation did not distinguish optional disk-TIFF compression from Zarr compression or semantic materialization.", + "The compression explanation did not contrast lossless codec selection with the existing value-changing callable dtype policy." + ], + "disposition": "revise", "validation": [ "Re-read docs/source/concepts/storage_system.rst in full against repository base bb4218fae and the current shared worktree; challenged its explanation user need for accuracy, freshness, duplication, owner boundaries, flow, and navigation.", "Compared every factual claim directly with the current hash-pinned authorities: openhcs/core/config.py; openhcs/core/runtime_stores.py; openhcs/core/steps/function_artifact_materialization.py; external/PolyStore/src/polystore/base.py; tests/unit/test_runtime_value_store.py.", @@ -917,7 +940,9 @@ "2026-09-09: Re-read this page against the changed source authorities for release 0.8.3; retained its Diataxis purpose and updated affected workflow claims.", "2026-09-16: Re-read the complete explanation against runtime artifact materialization changes. Added the semantic-versus-filename identity boundary and confirmed that path completion does not mutate artifact metadata; 249 focused materialization and adjacent runtime tests passed.", "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/config.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`.", - "2026-09-22 PR #126 integrated-repair follow-up: re-read the complete page and the final owner/test changes in `openhcs/core/steps/function_artifact_materialization.py`. Scalar image materialization now selects the execution scope matching the complete source address, deduplicates equivalent repeated group records, and rejects conflicts; the documented separation of semantic source identity, filename completion, runtime records, and persistence remains accurate. Refreshed only these compared evidence hashes; details are retained in `docs/audits/pr126_authority_delta_review_20260922.md`." + "2026-09-22 PR #126 integrated-repair follow-up: re-read the complete page and the final owner/test changes in `openhcs/core/steps/function_artifact_materialization.py`. Scalar image materialization now selects the execution scope matching the complete source address, deduplicates equivalent repeated group records, and rejects conflicts; the documented separation of semantic source identity, filename completion, runtime records, and persistence remains accurate. Refreshed only these compared evidence hashes; details are retained in `docs/audits/pr126_authority_delta_review_20260922.md`.", + "2026-09-24 PR #129: re-read the complete explanation against the changed config, materialization, PolyStore writer, and exact-roundtrip test authorities; added the disk-TIFF compression boundary without copying exact generated fields. Review scope is recorded in `docs/audits/pr129_tiff_compression_review_20260924.md`.", + "2026-09-24: re-read the complete explanation and checked the ArrayBridge dtype wrapper against the PolyStore TIFF writer; documented that the codec preserves the callable-produced dtype, while a callable-level conversion changes it before persistence." ] }, { @@ -932,7 +957,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Declares the package version, Python requirement, dependency extras, scripts, and distribution metadata." }, { @@ -1167,7 +1192,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the documentation-enabled Function Selector capture declaration, accessibility text, capture target, and generated visual projection." }, { @@ -1264,7 +1289,7 @@ }, { "path": "docs/source/guide_for_biologists/configuration_reference.rst", - "source_sha256": "3a70946f59a595f5ebbdf4a176280d7ba497f630cec92c45cd2710d40a10f6f2", + "source_sha256": "ba3c4ef923c4f643c06accc8641c462b2f63a8e46694ef984dde96ef49a0d9c7", "audience": [ "imaging scientists", "pipeline authors" @@ -1274,7 +1299,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -1324,13 +1349,14 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the documentation-enabled global, pipeline, step, and nested function configuration capture declarations, accessibility text, capture targets, and generated visual projections." } ], "findings": [ "The configuration explanation now distinguishes source-document omission, which selects PipelineConfig(), from normal lazy-field inheritance.", - "The explanation did not tell agents that live field help projects accepted enum input strings, including member names for non-string values." + "The explanation did not tell agents that live field help projects accepted enum input strings, including member names for non-string values.", + "The practitioner configuration map did not name the new optional TIFF compression family or distinguish its output scope from Zarr and viewer streams." ], "disposition": "revise", "validation": [ @@ -1347,7 +1373,8 @@ "2026-09-08: targeted 0.8.3 authority-delta review, retaining the earlier full-page review. Checked affected catalogue ownership, code round-trip, runtime-setting and endpoint lifecycle claims against the changed declarations. Updated the code-editing, dtype, callable-authoring and connection explanations where behaviour changed. Dependency and version changes do not alter the retained installation or package-ownership descriptions. Focused source/ObjectState round-trip and callable-contract checks passed 53 tests; this is not a new whole-document audit.", "2026-09-09: Re-read this page against the changed source authorities for release 0.8.3; retained its Diataxis purpose and updated affected workflow claims.", "2026-09-09: Reviewed the page sections affected by exact manager-item navigation, selection recovery guidance, and the pyqt-reactive dependency floor change from 0.3.20 to 0.3.21. Generic selection and deferred dispatch remain in pyqt-reactive; domain window identities stay in OpenHCS. Navigation acceptance does not replace action revision guards. Installation extras, Python support, code-document apply, and experimental/runtime semantics are unchanged by this batch. Updated the MCP architecture and basic-interface navigation explanations; retained existing setup/ownership links rather than duplicating a version catalogue. Only the reviewed navigation/dependency authorities were refreshed, excluding the scientific threshold patch. Final parent gates: 176 OpenHCS and 463 pyqt-reactive tests passed, including deferred target removal; full fresh-agent trial remains separate acceptance evidence.", - "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/config.py`, `tests/unit/pyqt_gui/test_ui_agent_bridge.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`." + "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/config.py`, `tests/unit/pyqt_gui/test_ui_agent_bridge.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`.", + "2026-09-24 PR #129: re-read the complete inheritance explanation against the new registered TIFF config and added its disk-output scope to the existing mental model; exact fields remain generated. Review scope is recorded in `docs/audits/pr129_tiff_compression_review_20260924.md`." ] }, { @@ -1461,7 +1488,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." } ], @@ -1530,7 +1557,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the documentation-enabled source-image browser capture, caption, accessibility text, dimensions, and source evidence." }, { @@ -1550,7 +1577,7 @@ }, { "path": "external/PolyStore/src/polystore/config.py", - "sha256": "241f728c991b9a4ac4d4440bba9bc08b5d89d23e7f8b3b9fcbdc18eb562a9bbc", + "sha256": "62c8acad3da3abed8ccf7f1166fc6a0d094be1c3e842cec6446780541df292a0", "role": "Owns the nominal Zarr compressor factories, Numcodecs creation, chunk strategy, and backend configuration." }, { @@ -1649,7 +1676,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Declares the package version, Python requirement, dependency extras, scripts, and distribution metadata." }, { @@ -1781,7 +1808,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the documentation-enabled synthetic generator, plate metadata, and live measurement result capture declarations used by the first-workflow tutorial." } ], @@ -2323,7 +2350,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Includes the complete openhcs/omero deployment and plugin bundle in installed distributions." }, { @@ -2805,7 +2832,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the landing-page workspace scenario, representative image, caption, description, alt text, dimensions, and release evidence." } ], @@ -2836,7 +2863,7 @@ "authority": [ { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Declares the package version, Python requirement, dependency extras, scripts, and distribution metadata." }, { @@ -2903,7 +2930,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -3085,7 +3112,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -3130,7 +3157,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -3192,7 +3219,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Projects every registered stable-window identity into its documentation scenario, capture target, filename, and accessibility text." }, { @@ -3240,7 +3267,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -3369,7 +3396,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the lazy-inheritance scenario, representative poster, caption, description, alt text, dimensions, and release evidence." }, { @@ -3469,7 +3496,7 @@ }, { "path": "pyproject.toml", - "sha256": "61621859d0f780d2d3e004782f242ef923d6cd1e83bb3bbc9761197ba33a4165", + "sha256": "6217b5f029796edcd83aa47fb8632d37aa7552344755c578fa4f181a6b23282d", "role": "Declares the package version, Python requirement, dependency extras, scripts, and distribution metadata." } ], @@ -3567,7 +3594,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the documentation-enabled Custom Function Manager capture declaration, accessibility text, capture target, and generated visual projection." }, { @@ -3686,7 +3713,7 @@ }, { "path": "docs/source/user_guide/dtype_conversion.rst", - "source_sha256": "15887645e3b1f7ffc098364060108c42fcd72f2be9de9968ece8f3610ed74bd1", + "source_sha256": "c95144eee07fe4134768fd2e839cb25dfaa9d61e9794332c8481ccfc39a95d4c", "audience": [ "pipeline authors" ], @@ -3695,7 +3722,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -3708,6 +3735,21 @@ "sha256": "99823105b0ae59054df1d84013633b8b6628d0a6dc3326383199ac7c65c69bed", "role": "Declares decorators that write independent input, output, and execution memory roles plus dtype policy." }, + { + "path": "external/arraybridge/src/arraybridge/array_operations.py", + "sha256": "7e9ebda3d058702cdf8662dc5ab1faf1254649f3cbde76fa0adcbe02677fa103", + "role": "Defines NumPy float-to-integer scaling and integer-to-integer casting at the callable dtype boundary." + }, + { + "path": "openhcs/processing/backends/processors/numpy_processor.py", + "sha256": "311634ec55ec8ddefe05118329d208e4b33df86f81d54ca4e258e7f4d2dbc980", + "role": "Declares the decorated projection function used in the 8-bit materialisation example." + }, + { + "path": "external/PolyStore/src/polystore/disk.py", + "sha256": "cf089b8c79f86099a7cea1b6cbe43ceaec6331196a44b43183f2e64b7746c219", + "role": "Writes materialised TIFF pixels at the dtype supplied by the processing result." + }, { "path": "tests/unit/test_callable_contract.py", "sha256": "493f76a54f5fe024ca8f10c53b1f810b8c369626ad7141205b42ef3a78282137", @@ -3720,7 +3762,8 @@ } ], "findings": [ - "The task guide told users to change a desktop or step setting without naming the control or showing an executable override." + "The task guide told users to change a desktop or step setting without naming the control or showing an executable override.", + "The callable override lacked a concrete 8-bit materialisation example and warnings about NumPy scaling, integer wrapping, downstream values, and auxiliary outputs." ], "disposition": "revise", "validation": [ @@ -3731,7 +3774,8 @@ "Passed ./.venv/bin/python -m sphinx -E -W --keep-going -b html docs/source : 153 source files rendered without warnings.", "On 2026-08-24, re-read the complete page against the changed declaration, dependency, UI state, knowledge, and test authorities named by this record; the documented behavior and ownership boundaries remain accurate.", "2026-09-08: targeted 0.8.3 authority-delta review, retaining the earlier full-page review. Checked affected catalogue ownership, code round-trip, runtime-setting and endpoint lifecycle claims against the changed declarations. Updated the code-editing, dtype, callable-authoring and connection explanations where behaviour changed. Dependency and version changes do not alter the retained installation or package-ownership descriptions. Focused source/ObjectState round-trip and callable-contract checks passed 53 tests; this is not a new whole-document audit.", - "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/config.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`." + "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/config.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`.", + "2026-09-24: re-read the full how-to and verified the new callable-level UINT8 materialisation example instantiates in the source checkout; direct NumPy-backed projection checks confirmed float scaling and integer wrapping. Reviewed ArrayBridge wrapper, NumPy scaling, the example callable, and the disk TIFF writer." ] }, { @@ -3755,7 +3799,7 @@ }, { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the default directory filenames and replicate-local normalization method." }, { @@ -3878,7 +3922,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the documentation-enabled execution-log capture, caption, accessibility text, dimensions, and source evidence." } ], @@ -3950,7 +3994,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the local-MCP endpoint-startup scenario, representative poster, caption, description, alt text, dimensions, and release evidence." }, { @@ -4048,7 +4092,7 @@ "authority": [ { "path": "openhcs/core/config.py", - "sha256": "628b338097814f3d0297c28da8d016252214ff2253aee2e4946bad09f5890ea9", + "sha256": "ef81b50ee22b8da69513fd50d6e45a929117fa4bfaa62a20ef08a4c02531c043", "role": "Declares the supported OpenHCS configuration dataclasses, nested scopes, defaults, and field help metadata." }, { @@ -4098,7 +4142,7 @@ }, { "path": "scripts/gallery_catalog.py", - "sha256": "e6516488d7024771c35faf1a25d3ec3abc3b2f21830a8b318c9665d9469c5c0a", + "sha256": "bda42a36c1c510af7ac587dc2642548758d28c4a0061d581ebe38a5be4fb3743", "role": "Owns the Napari ROI-navigation and Fiji ROI-review scenarios, representative images, captions, descriptions, alt text, dimensions, and release evidence." } ], diff --git a/docs/source/concepts/storage_system.rst b/docs/source/concepts/storage_system.rst index 23a0effbda..66f05d5944 100644 --- a/docs/source/concepts/storage_system.rst +++ b/docs/source/concepts/storage_system.rst @@ -26,6 +26,23 @@ enum. OpenHCS contributes a fieldless registered subtype so those exact storage identities participate in global, pipeline, and step configuration without a second enum or field declaration. +Disk TIFF compression +--------------------- + +``TiffConfig`` controls lossless compression for disk-backed TIFF outputs, +including ordinary main-flow images and explicitly materialized named images. +It is separate from Zarr compression and does not change non-TIFF artifacts or +viewer streams. The default ``NONE`` retains uncompressed output; ``DEFLATE`` +uses the declared ``compression_level`` from 1 to 9. PolyStore owns the codec +and writer behavior, while OpenHCS resolves the configuration for a pipeline +and passes it to disk materialization. See :doc:`../reference/configuration` +for exact fields and defaults. + +Compression preserves the dtype and pixel values produced by the callable. +For a smaller 8-bit display image, set the existing callable-level dtype policy +on the image-producing function instead; this changes values before storage. +See :doc:`../user_guide/dtype_conversion` for the conversion boundaries. + Artifacts first, paths second ----------------------------- diff --git a/docs/source/guide_for_biologists/configuration_reference.rst b/docs/source/guide_for_biologists/configuration_reference.rst index 033b6a574e..801fbdeadf 100644 --- a/docs/source/guide_for_biologists/configuration_reference.rst +++ b/docs/source/guide_for_biologists/configuration_reference.rst @@ -120,6 +120,13 @@ Storage, paths, and persistence Read, intermediate, and persistent storage choices plus Zarr compression and chunking. +``TiffConfig`` + Optional lossless compression for disk-backed TIFF output, including saved + main-flow images and named image artifacts. The default is uncompressed; + enabling ``DEFLATE`` does not affect Zarr, other file formats, or viewer + streaming. The codec and level are resolved with the pipeline configuration + before execution. + ``PathPlanningConfig`` Output workspace/root placement, directory naming, and which processed wells seed the automatic main-flow output plate. ``well_filter=0`` keeps that diff --git a/docs/source/user_guide/dtype_conversion.rst b/docs/source/user_guide/dtype_conversion.rst index 0f020902d6..a410a1cb63 100644 --- a/docs/source/user_guide/dtype_conversion.rst +++ b/docs/source/user_guide/dtype_conversion.rst @@ -27,13 +27,59 @@ The equivalent step declaration is: .. code-block:: python - from arraybridge.decorators import DtypeConversion from openhcs.core.config import LazyDtypeConfig + from arraybridge.decorators import DtypeConversion step.dtype_config = LazyDtypeConfig( default_dtype_conversion=DtypeConversion.PRESERVE_INPUT, ) +Save an 8-bit visualisation +--------------------------- + +For a display image that does not need its original precision, set ``UINT8`` +on the image-producing callable in the function pattern and materialise that +step's result: + +.. code-block:: python + + from openhcs.core.config import LazyDtypeConfig, LazyStepMaterializationConfig + from openhcs.core.steps.function_step import FunctionStep + from openhcs.processing.backends.processors.numpy_processor import create_projection + from arraybridge.decorators import DtypeConversion + + preview_step = FunctionStep( + func=( + create_projection, + { + "dtype_config": LazyDtypeConfig( + default_dtype_conversion=DtypeConversion.UINT8, + ), + }, + ), + name="8-bit projection for review", + step_materialization_config=LazyStepMaterializationConfig(enabled=True), + ) + +Use this example with a floating-point image stack. The callable-level policy +converts the main result before materialisation. When TIFF is the selected +output format, the disk file therefore contains 8-bit pixels. +The policy belongs to this callable occurrence, not to every function in the +pipeline. Later steps using this result also receive its 8-bit values, so keep +the conversion on a terminal review route if analysis still needs full +precision. + +Check the values before choosing this policy. For NumPy-backed callables, +float-to-``UINT8`` conversion scales the minimum and maximum of each +non-constant callable result to the 8-bit range. Integer-to-``UINT8`` +conversion casts instead: values outside 0--255 can wrap. Neither is suitable +for preserving quantitative intensities or object-label identities. A +tuple-returning callable converts only its first, main result; it does not +convert auxiliary named outputs. + +To save space without changing pixel values, use lossless disk TIFF compression +instead. See :doc:`../concepts/storage_system` for the separate storage policy. + Compile again after changing the policy. Do not cast merely to cross NumPy/CuPy/Torch boundaries; memory conversion and dtype policy are separate declarations. diff --git a/external/PolyStore b/external/PolyStore index 6534b10e7a..0efe67fdd1 160000 --- a/external/PolyStore +++ b/external/PolyStore @@ -1 +1 @@ -Subproject commit 6534b10e7a4dc077fe4d0a710bb05b995ee1ba57 +Subproject commit 0efe67fdd14985bf90cee6e0f0c4735d41d265d1 diff --git a/openhcs/core/config.py b/openhcs/core/config.py index 8aec5091bc..8aaa01daba 100644 --- a/openhcs/core/config.py +++ b/openhcs/core/config.py @@ -69,6 +69,7 @@ class StreamingConfigMeta(EnableableMeta, AutoRegisterMeta): Lz4ZarrCompressorFactory = _polystore_config.Lz4ZarrCompressorFactory ZstdZarrCompressorFactory = _polystore_config.ZstdZarrCompressorFactory ZarrChunkStrategy = _polystore_config.ZarrChunkStrategy +TiffCompression = _polystore_config.TiffCompression class MaterializationBackend(Enum): @@ -616,6 +617,16 @@ class ZarrConfig(AnnotatedDataclassValidationMixin, _polystore_config.ZarrConfig """ +@abbreviation("tiff") +@global_pipeline_config( + inherit_as_none=False, + field_abbreviations={"compression": "codec", "compression_level": "level"}, +) +@dataclass(frozen=True) +class TiffConfig(AnnotatedDataclassValidationMixin, _polystore_config.TiffConfig): + """Lossless TIFF output settings shared by image and artifact materialization.""" + + @abbreviation("vfs") @global_pipeline_config(always_viewable_fields=["materialization_backend"]) @dataclass(frozen=True) diff --git a/openhcs/core/context/processing_context.py b/openhcs/core/context/processing_context.py index ac44a2abab..e9a6f47172 100644 --- a/openhcs/core/context/processing_context.py +++ b/openhcs/core/context/processing_context.py @@ -19,6 +19,7 @@ AnalysisConsolidationConfig, PlateMetadataConfig, StreamingConfig, + TiffConfig, ) from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.debug import ( @@ -97,6 +98,7 @@ def __init__( filemanager: FileManager | None = None, analysis_consolidation_config: AnalysisConsolidationConfig | None = None, plate_metadata_config: PlateMetadataConfig | None = None, + tiff_config: TiffConfig | None = None, auto_add_output_plate_to_plate_manager: bool = False, output_plate_root: str | None = None, transport_config: ZMQConfig = OPENHCS_ZMQ_CONFIG, @@ -147,6 +149,7 @@ def __init__( if plate_metadata_config is not None else PlateMetadataConfig() ) + self.tiff_config = tiff_config if tiff_config is not None else TiffConfig() self.auto_add_output_plate_to_plate_manager = ( auto_add_output_plate_to_plate_manager ) diff --git a/openhcs/core/orchestrator/orchestrator.py b/openhcs/core/orchestrator/orchestrator.py index 22ed3ea28a..09d3836fb2 100644 --- a/openhcs/core/orchestrator/orchestrator.py +++ b/openhcs/core/orchestrator/orchestrator.py @@ -574,6 +574,7 @@ def create_context(self, axis_id: str) -> ProcessingContext: effective_config.analysis_consolidation_config ), plate_metadata_config=effective_config.plate_metadata_config, + tiff_config=effective_config.tiff_config, auto_add_output_plate_to_plate_manager=( effective_config.auto_add_output_plate_to_plate_manager ), diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index d27788c823..6bfc6e5b63 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -405,7 +405,12 @@ def backend_kwargs( f"context values: {sorted(conflicts)!r}." ) values.update(contextual_values) - result[backend] = RawBackendKwargs(values) + result[backend] = RawBackendKwargs( + values, + tiff_config=( + kwargs.tiff_config if isinstance(kwargs, RawBackendKwargs) else None + ), + ) streamable_viewer_surfaces = self.streamable_viewer_surfaces( filemanager=filemanager, @@ -496,7 +501,7 @@ def backend_plan( ) ) return ArtifactMaterializationBackendPlan( - persistent_backend_kwargs=self.persistent_backend_kwargs(), + persistent_backend_kwargs=self.persistent_backend_kwargs(context), streaming_viewer_surfaces=( { config.backend.value: config.streaming_viewer_surface(context) @@ -514,7 +519,7 @@ def backend_plan( ) @abstractmethod - def persistent_backend_kwargs(self) -> BackendKwargs: + def persistent_backend_kwargs(self, context: "ProcessingContext") -> BackendKwargs: """Return persistent materialization backends owned by this policy.""" @@ -525,8 +530,14 @@ class PersistentArtifactMaterializationTargetPlan(ArtifactMaterializationTargetP target_key = "persistent" backend: str - def persistent_backend_kwargs(self) -> BackendKwargs: - return {self.backend: RawBackendKwargs()} + def persistent_backend_kwargs(self, context: "ProcessingContext") -> BackendKwargs: + return { + self.backend: RawBackendKwargs( + tiff_config=( + context.tiff_config if self.backend == Backend.DISK.value else None + ), + ) + } class StreamingOnlyArtifactMaterializationTargetPlan(ArtifactMaterializationTargetPlan): @@ -534,7 +545,7 @@ class StreamingOnlyArtifactMaterializationTargetPlan(ArtifactMaterializationTarg target_key = "streaming_only" - def persistent_backend_kwargs(self) -> BackendKwargs: + def persistent_backend_kwargs(self, context: "ProcessingContext") -> BackendKwargs: return {} diff --git a/openhcs/core/steps/function_io.py b/openhcs/core/steps/function_io.py index 64236b7bf6..d2e5e5e38b 100644 --- a/openhcs/core/steps/function_io.py +++ b/openhcs/core/steps/function_io.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Callable, ClassVar, Mapping, Sequence, TypeAlias from metaclass_registry import AutoRegisterMeta +from polystore.config import tiff_write_batches from polystore.zarr_batch import ZarrBatchAxis, ZarrBatchAxisRole, ZarrBatchLayout from openhcs.constants.constants import ( @@ -380,9 +381,17 @@ def save_materialized_data( materialized_paths, materialized_backend, ) - filemanager.save_batch( - payloads, list(materialized_paths), materialized_backend, **save_kwargs + tiff_config = ( + context.tiff_config if materialized_backend == Backend.DISK.value else None ) + for indices, batch_config in tiff_write_batches(materialized_paths, tiff_config): + filemanager.save_batch( + [payloads[index] for index in indices], + [materialized_paths[index] for index in indices], + materialized_backend, + **save_kwargs, + **({"tiff_config": batch_config} if batch_config is not None else {}), + ) def get_all_image_paths( diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index c8733dc8d5..437f42ab19 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -22,6 +22,7 @@ import numpy as np import pandas as pd from metaclass_registry import AutoRegisterMeta +from polystore.config import TiffConfig, tiff_write_batches from polystore.streaming.viewer_transport import ( PathMappedViewerStreamSourceMetadata, ViewerStreamBackendKwargs, @@ -158,6 +159,7 @@ class RawBackendKwargs(BackendCallKwargs, Mapping[str, MaterializationValue]): backend_kwargs_kind = "raw" values: Mapping[str, MaterializationValue] = field(default_factory=dict) + tiff_config: TiffConfig | None = None def __getitem__(self, key: str) -> MaterializationValue: return self.values[key] @@ -172,13 +174,29 @@ def to_filemanager_kwargs( self, output: Output, ) -> dict: - return dict(self.values) + values = dict(self.values) + if self.tiff_config is not None and self.tiff_config.applies_to_path( + output.path + ): + values["tiff_config"] = self.tiff_config + return values def filemanager_batches( self, outputs: Sequence[Output], ) -> tuple[tuple[tuple[Output, ...], dict], ...]: - return ((tuple(outputs), dict(self.values)),) + return tuple( + ( + tuple(outputs[index] for index in indices), + { + **self.values, + **({"tiff_config": config} if config is not None else {}), + }, + ) + for indices, config in tiff_write_batches( + tuple(output.path for output in outputs), self.tiff_config + ) + ) class TabularRow(dict[str, MaterializationValue]): diff --git a/pyproject.toml b/pyproject.toml index b20d05c0c7..cede0e82e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ dependencies = [ "python-introspect>=0.1.14,<0.2", "metaclass-registry>=0.2.1,<0.3", "arraybridge>=0.3.4,<0.4", - "polystore>=0.2.18,<0.3", + "polystore>=0.2.19,<0.3", "pyqt-reactive>=0.3.24,<0.4", ] diff --git a/scripts/gallery_catalog.py b/scripts/gallery_catalog.py index c445344ca6..701d45636b 100644 --- a/scripts/gallery_catalog.py +++ b/scripts/gallery_catalog.py @@ -25,7 +25,6 @@ from pyqt_reactive.services.system_monitor_actions import SystemMonitorAction from python_introspect import dataclass_from_mapping -from openhcs.agent.dto.execution import ExecutionConnectionSpec from openhcs.agent.ui_bridge_actions import PlateManagerAction from openhcs.agent.ui_bridge_identities import ( GlobalConfigWindowIdentity, @@ -41,6 +40,7 @@ from openhcs.serialization.json import JsonValue, to_jsonable if TYPE_CHECKING: + from openhcs.agent.dto.execution import ExecutionConnectionSpec from openhcs.pyqt_gui.services.step_scope_identity import StepEditorScope RELEASE_MEDIA_SCHEMA_VERSION = "openhcs.release-media.v7" diff --git a/tests/unit/test_config_document.py b/tests/unit/test_config_document.py index 57621a2d92..157caefc62 100644 --- a/tests/unit/test_config_document.py +++ b/tests/unit/test_config_document.py @@ -7,7 +7,12 @@ import pytest -from openhcs.core.config import GlobalPipelineConfig, PipelineConfig +from openhcs.core.config import ( + GlobalPipelineConfig, + PipelineConfig, + TiffCompression, + TiffConfig, +) from openhcs.core.config_document import ( ConfigDocumentAuthority, ConfigDocumentField, @@ -27,6 +32,12 @@ def test_config_document_field_owns_exact_public_name() -> None: "config", ( GlobalPipelineConfig(num_workers=3), + GlobalPipelineConfig( + tiff_config=TiffConfig( + compression=TiffCompression.DEFLATE, + compression_level=3, + ) + ), PipelineConfig(), ), ) diff --git a/tests/unit/test_function_artifact_materialization.py b/tests/unit/test_function_artifact_materialization.py index dfdb539877..fce6077a7e 100644 --- a/tests/unit/test_function_artifact_materialization.py +++ b/tests/unit/test_function_artifact_materialization.py @@ -43,7 +43,7 @@ RuntimeExecutionAxisScope, ) from openhcs.core.components.parser_metaprogramming import FilenameParseResult -from openhcs.core.config import AnalysisConsolidationConfig, WellFilterMode +from openhcs.core.config import AnalysisConsolidationConfig, TiffConfig, WellFilterMode from openhcs.core.function_patterns import ( DEFAULT_GROUP_KEY, CompiledFunctionGroup, @@ -468,6 +468,7 @@ def _context(filemanager): context.axis_id = "A01" context.step_axis_filters = {} context.analysis_consolidation_config = AnalysisConsolidationConfig() + context.tiff_config = TiffConfig() return context diff --git a/tests/unit/test_function_artifact_materialization_io_hotpath.py b/tests/unit/test_function_artifact_materialization_io_hotpath.py index d8fe1aa816..b6627f4cfa 100644 --- a/tests/unit/test_function_artifact_materialization_io_hotpath.py +++ b/tests/unit/test_function_artifact_materialization_io_hotpath.py @@ -7,6 +7,7 @@ import numpy as np import pytest from polystore.base import DataSink +from polystore.config import TiffCompression, TiffConfig from polystore.streaming.identity import StreamProducerIdentity from openhcs.core.steps.function_artifact_materialization import ( @@ -15,6 +16,7 @@ ) from openhcs.processing.materialization.core import ( MaterializationSpec, + Output, RawBackendKwargs, ) from openhcs.processing.materialization.options import ImageFileOptions @@ -38,6 +40,26 @@ def save_batch(self, data_list, identifiers, **kwargs): raise AssertionError("This projection test must not save") +def test_persistent_tiff_kwargs_apply_only_to_tiff_outputs() -> None: + config = TiffConfig(compression=TiffCompression.DEFLATE) + kwargs = RawBackendKwargs(tiff_config=config) + outputs = ( + Output(path="/results/summary.csv", content="well,count\nA01,1\n"), + Output(path="/results/labels.tif", content=np.zeros((8, 8), dtype=np.int32)), + Output(path="/results/labels2.tiff", content=np.zeros((8, 8), dtype=np.int32)), + ) + + batches = kwargs.filemanager_batches(outputs) + + assert tuple(output.path for output in batches[0][0]) == ("/results/summary.csv",) + assert batches[0][1] == {} + assert tuple(output.path for output in batches[1][0]) == ( + "/results/labels.tif", + "/results/labels2.tiff", + ) + assert batches[1][1] == {"tiff_config": config} + + class _FileManager: def __init__(self) -> None: self.backend = _PersistentBackend() diff --git a/tests/unit/test_tiff_output_compression.py b/tests/unit/test_tiff_output_compression.py new file mode 100644 index 0000000000..7ada403ad4 --- /dev/null +++ b/tests/unit/test_tiff_output_compression.py @@ -0,0 +1,72 @@ +"""Lossless TIFF compression crosses the declared OpenHCS output boundary.""" + +from types import SimpleNamespace + +import numpy as np +import tifffile +from polystore.disk import DiskBackend +from polystore.filemanager import FileManager + +from openhcs.constants.constants import Backend +from openhcs.core.config import GlobalPipelineConfig, TiffCompression, TiffConfig +from openhcs.core.steps.function_artifact_materialization import ( + PersistentArtifactMaterializationTargetPlan, +) +from openhcs.core.steps.function_io import save_materialized_data +from openhcs.processing.materialization import ( + ImageFileOptions, + MaterializationSpec, + MaterializedFilenameIdentity, + materialize, +) + + +def test_configured_main_flow_tiff_output_is_lossless(tmp_path) -> None: + config = TiffConfig(compression=TiffCompression.DEFLATE, compression_level=3) + assert GlobalPipelineConfig().tiff_config == TiffConfig() + pixels = np.zeros((2, 256, 256), dtype=np.int32) + pixels[0, 40:60, 80:100] = 5 + path = tmp_path / "labels.tif" + context = SimpleNamespace( + microscope_handler=SimpleNamespace(parser=object(), microscope_type="test"), + tiff_config=config, + ) + + save_materialized_data( + FileManager({Backend.DISK.value: DiskBackend()}), + [pixels], + [str(path)], + Backend.DISK.value, + None, + context, + "A01", + ) + + with tifffile.TiffFile(path) as output: + assert output.pages[0].compression.value != 1 + np.testing.assert_array_equal(tifffile.imread(path), pixels) + + +def test_configured_named_artifact_tiff_output_is_lossless(tmp_path) -> None: + config = TiffConfig(compression=TiffCompression.DEFLATE, compression_level=3) + pixels = np.zeros((2, 256, 256), dtype=np.int32) + pixels[1, 60:80, 90:110] = 11 + path = materialize( + MaterializationSpec( + ImageFileOptions( + filename_suffix=".tif", + filename_identity=MaterializedFilenameIdentity.ARTIFACT_NAME, + ) + ), + pixels, + str(tmp_path / "artifact"), + FileManager({Backend.DISK.value: DiskBackend()}), + backends=[Backend.DISK.value], + backend_kwargs=PersistentArtifactMaterializationTargetPlan( + backend=Backend.DISK.value + ).persistent_backend_kwargs(SimpleNamespace(tiff_config=config)), + ) + + with tifffile.TiffFile(path) as output: + assert output.pages[0].compression.value != 1 + np.testing.assert_array_equal(tifffile.imread(path), pixels)