diff --git a/docs/validation/opera_producer_inventory_172.rst b/docs/validation/opera_producer_inventory_172.rst new file mode 100644 index 0000000000..2fe74656b6 --- /dev/null +++ b/docs/validation/opera_producer_inventory_172.rst @@ -0,0 +1,435 @@ +Synthetic acquisition identity/calibration, issues 172/335 +========================================================= + +Source owner: Averroes. Parent owns integration, installation and actual MCP +qualification. This new task explicitly transfers the former parent-owned +A01/R01C01 producer follow-up; PR 329/XML parser stays frozen and unchanged. +Dispatch read completely: OPERA-INVENTORY-DISPATCH-20261001.rst in the parent's +issue-batch worktree. Base/current main: b31807d9de067e18d212bdf9353d05099757fc74. +Own worktree: /home/ts/wt/openhcs-opera-producer-inventory-172-20261001. +Branch: fix/opera-producer-inventory-172-20261001. + +Guidance and ownership before edits +---------------------------------- + +Read complete current nra-refactoring and local refactor-audit SKILL.md; +authoritative refactor-audit.skill ZIP entrypoint, pattern README, surface +receipt, complete identity/boundaries/membership/implementation/over-time +references before census or decisions. Archive SHA256: +100fbe8ef89664b866777e87b2c8640a3432e8a10e9188dff81c97942d551bf6. +NRA skill SHA256: +9f2f8b28bc82256eefa3e9d63248c50722dc3ffe7d77adba5793296df196b47e. +Standing user-provided AGENTS instructions apply; no on-disk AGENTS found here. + +Read active PR projections: 329 owns XML geometry; 331 owns compiler/image +carrier admission; 326 owns runtime optimizations. Arendt owns pipeline MCP +renderers. None claims the producer/parser files selected here. No foreign +worktree, installed package, frozen blind input/output or original fixture edits. + +IDEN-3/IDEN-6: SyntheticMicroscopyGenerator.generate_openhcs_metadata:1125 +builds A01 catalog keys from requests while OperaPhenixFilenameParser:488 parses +R01C01. Display/request labels are not semantic catalog membership. +ImageXpress stacks can repeat basenames in ZStep folders; metadata flattened +basename keys then repeat/overwrite references. One plane must retain its exact +well/site/channel/Z/time address AND its physical source reference. + +BOUND-2: traced FilenameParseResult/GenericFilenameParser/FilenameParser; +actual ImageXpress/Opera constructors; MicroscopeHandler's AutoRegisterMeta +registry and parser association; OpenHCSMetadata/MetadataComponentValueSet; +OpenHCSPlaneAddress, SourcePlaneProjection, SourceProjectionSet and the richer +SourceProjectionMetadataSerializer. The serializer already owns canonical +filename round-trip validation, declaration-derived component membership and +all coherent path-keyed source fields. Reuse it, not a second metadata collector. + +IMPL-1/IMPL-5/IMPL-12: move acquisition spelling variation to existing parser +leaves with shared behavior on their nominal ancestor. The generator uses one +typed write/retained-projection operation; no string/type switch for filenames +or metadata. Decode the requested format once through existing registered +handler declarations, deriving parser and microscope names from that instance. +Unrelated existing XML/HTD/grid-layout branches are not copied or expanded. + +MEMB-1/MEMB-4/MEMB-5: no mirror enum/class roster; names derive from actual +registered declarations. Catalogs and refs derive from emitted typed projections, +not requested counts or a separately maintained dict shape. No new production +class is needed. Existing parser inheritance supplies the shared operation; +concrete owners supply acquisition spelling hooks. MI/MRO is respected without +inventing overlapping capabilities unsupported by these two lexical formats. + +TIME-3/TIME-5/TIME-9: delete replaced generator filename helper/metadata path; +no aliases, reader converter, fallback, product test switch or parallel writer. +Stored format remains the existing OpenHCS source-projection contract. Original +broken persisted plates are evidence and remain unchanged, not silently migrated. + +Required relation and new-case experiment +---------------------------------------- + +Every actually saved plane contributes one validated complete address and disk +SourcePixelRef. The parser's canonical emitted spelling determines its well +identity. Physical acquisition names may omit scalar tokens or encode Z in +folders; complete retained addresses and canonical virtual names must remain +distinct, with an exact reference to each physical file. Metadata membership +comes only from saved planes, including skip-file controls. Labels cannot add +another well. All five AllComponents members and source fields move together. + +Executed new cases: D12/Z99 Opera wells, non-contiguous site subsets, omitted acquisition +tokens, multiple channels/Z, Bio-Formats physical naming, explicit custom virtual +subdirectory and repeat generation. The projection serializer handles these +without adding metadata cases or another identity formatter. + +Resources and exact proof boundary +---------------------------------- + +Owned disposable scratch: /home/ts/.cache/agent-scratch/opera-producer-inventory-172-20261001. +Purpose: bounded source imports, tiny synthetic controls, original errors and +R0/R1/census evidence. Reuse the prior archived disposable supervisor rather +than create another process mechanism. One CPU, 512 MiB process-group RSS, +60 seconds per shard, 256 MiB scratch. Resource guard before tests: RAM 19.2 GiB, +home 22.5 GiB; historical swap 12.5 GiB warning, not a veto under these bounds. +Latest archived guard: RAM 19.3 GiB, home 20.8 GiB, historical swap 12.4 GiB, +exit 2/warning. No guard failure is rewritten as success. +No dependency/environment downloads, installation, MCP, native pipeline, JVM, +Fiji, science slot/lock or biological dataset use. Existing Python 3.12 packages +and shared read-only source dependencies only; assert product source origins. + +Focused evidence is not complete NRA or installed readiness. Original guards +run unchanged, with failures/timeouts preserved and no exceptions or waivers. +Parent's remaining acceptance: raw and native Opera generation, complete empty +document initialization, sample/reopen and combined inventory through installed +entrypoint after slot release; one semantic well and coherent other components +and refs. PR 329's installed geometry check remains parent-owned separately. + +First working source checkpoint +------------------------------- + +Six real-import unittest methods pass (including sixteen raw/native, format, +Z-stack and explicit-token subcases); 2.41 s, 93.24 MiB combined RSS, no limit +hit. Tiny 32x32 disk controls verify all five component catalogs, canonical +filename/address equality, persisted source-projection/ref equality, unique +physical sources, real TIFF reads and fresh real DiskStorageBackend/FileManager +metadata-handler reopen. Bio-Formats Z folders, skipped channel and repeat +generation are covered. This is source-only evidence, not installed MCP. + +Before edits the same focused family failed twenty assertions/subcases, +including A01 != R01C01 and flattened Z basename/ref collisions. Preserve the +baseline traceback. The first working test harness incorrectly supplied None +to a metadata reader requiring FileManager; preserve that error too. Corrected +the harness to use the real disk owner, without mocks or reader changes. + +Imports: product generator/projection/parser source from this new worktree; +PolyStore read-only source from the earlier qualified namespace worktree at +91fd7e854ca760a16e7472d8d558b4012e89b6ba (same production source/tests as merged +PR 16/84f322). Existing other Python 3.12 dependencies are read-only. No source +or dependency pins are changed outside this new worktree. + +Cooperative inheritance and actual new-case proof +------------------------------------------------ + +The binding owner requirement to maximize inheritance/polymorphism is applied +within this assigned surface. GenericFilenameParser owns the shared acquisition +operation and complete declaration binding. FilenameParser already composes the +viewer parser ABC and that generic ancestor. Existing ImageXpress/Opera leaves +own physical spelling variation; the generator calls their nominal contract. +There is no new production class, alternative roster, sibling-as-subtype, +capability priority table, consumer conversion or stored compatibility path. + +The concrete regression test +test_new_nominal_capabilities_cooperate_through_real_parser_diamond adds two +independent test capabilities deriving from the real GenericFilenameParser ABC: +RecordedAcquisition owns a per-instance physical-name call ledger; +ScopedAcquisition owns a physical acquisition prefix. Both call super(), compose +with ImageXpressFilenameParser and share the genuine existing ancestor diamond. +Neither implements or copies component decoding/formatting or metadata logic. + +Both RecordedScopedImageXpressParser and ScopedRecordedImageXpressParser execute +the capabilities once. Their different cooperative orders produce the same +physical name scope172_D12_s017_w7.tif; the ledger records the scoped/unscoped +result at the correct MRO position. GenericFilenameParser occurs once in both +MROs, its declaration initialization supplies tuple(AllComponents), and its +canonical operation yields D12_s017_w7_z099_t003.tif with every axis preserved. +Automatic AutoRegisterMeta registration resolves both new parser classes. +Unchanged SourceProjectionMetadataSerializer, persisted projection decoder and +fresh real disk metadata reader retain exact refs, Z=99 and timepoint=3. + +An additional declared RecordedScopedImageXpressHandler inherits the genuine +ImageXpress handler and binds the new composed parser. Its existing automatic +handler registry accepts the declaration. The unchanged generator constructor +selects it; unchanged _write_plane and generate_openhcs_metadata emit/reopen the +scoped physical source and complete inventory. There are zero consumer or +registry edits for this new case. This proves the assigned plane-emission and +metadata boundary, not arbitrary new-family HTD or spatial-layout generation. +All five test-only declarations stay in the regression file; none is a new +production wrapper or alternate registry. + +Qualified source, original guards and exact coverage +--------------------------------------------------- + +Draft PR 333: https://github.com/OpenHCSDev/openhcs/pull/333 . Issue 172 OPEN. +Normally merged parent-qualified PR 329/main +2cbfc4a0771bf24d153b076bff4a5e321ac039f4 at working checkpoint +6d81dfd9f420970172d4e379d7cbed34c3d9fa01. No conflict or edits to frozen XML +parser: its bytes equal current main. The four owned production files are +unchanged by that merge. External gitlinks, scripts and benchmark equal main. + +Twenty-three source-contract methods pass after the merge: this task's twelve +(sixteen family subcases plus the additional controls and genuine MI/MRO +experiment) and PR 329's eleven unchanged XML geometry controls. 2.68 s, +94.27 MiB combined process-group RSS. Existing regression shard: 34 passed, +14.09 s, 385.61 MiB, covering zero geometry, MT19937 byte parity/caller isolation, +Bio-Formats lexical names, source projections and real tiny-disk virtual +workspace loads. No scientific pipeline, JVM or installed entrypoint is used. +No process group hits a resource limit. + +Public import evidence asserts all four owned source module origins in this +worktree, and the qualified read-only PolyStore origin; other dependencies come +from the existing Python 3.12 installation. Read-only Git comparison confirms +PolyStore 91fd versus merged 84f322 has identical src/tests. Baseline reproducer +uses its own clean detached Git worktree at b31807d9, one 32x32 physical plane, +and the actual original producer. Its persisted A01 and parsed R01C01 yield two +combined keys and the preserved failing assertion. Its XML/TIFF/JSON and +traceback are archived. Parent's original installed raw Opera XML retains +SHA256 b3f2c03c8e7b6e7e3196535b934f01f65c7c6b6f6bb22cf6fd6fd308e54db276. +Original parent fixtures and frozen blind trial remain untouched. + +Unchanged CI R0 ratchet from agent-comms +3b03785f45df2ef5dc62ba6aed99294192ecbb01 passes openhcs/scripts/benchmark +against merged main 2cbfc4a0 and working head 6d81dfd9. Tool SHA256 +e323c94d49c2b72d9524a5169f123e64b4a6e46a41035ca9fb4497e49b6ca562. +Root shards: openhcs 23.28 s/74.99 MiB; scripts 2.47 s/53.61 MiB; +benchmark 3.22 s/51.23 MiB; all exit 0. The only nonzero guard delta is +SyntheticMicroscopyGenerator GodClassExcess -140; every other guard delta zero. +This is deletion in place, not relocation, detector changes or exemptions. + +Current archive's untouched focused census measures all four changed production +files with no exclusions and zero unparsed files. String equality -6, optional +None checks +7, long function -1, code lines -52; all dispatch/type-switch/raw-key +and other measures have zero delta. Optional acquisition spelling parameters +and fail-closed parser admission explain the new None checks; they are reported, +not hidden. AST evidence parses original/current source, hashes all four files, +proves metadata/plane operations have no format dispatch and the obsolete +generator filename helper is deleted. This is not semantic descent proof. + +Original R1 policy consumer attempted unchanged, using its workflow-pinned NRA +0844525ecaba93e090a064a4ae4914466b2dae60 from a new read-only tool worktree, +full default production roots and 50 s budget. It exits 1 in 5.23 s/338.38 MiB +before analysis: Recorded source repository is not initialized: +external/ObjectState. The exact traceback is preserved. All eight new source +worktree gitlinks remain deliberately uninitialized; no dependency context is +excluded, invented, borrowed as Git metadata or downloaded to fabricate a pass. +Global R1 schema/descent certificates are unresolved. Focused evidence is not a +full NRA scan, R1 pass, waiver or installed readiness claim. + +Remaining acceptance and owned cleanup +------------------------------------- + +Parent reports actual installed PR 329 singleton MCP journey passed and merged. +That is parent evidence, separate from the source checks above. Parent now owns +building/installing this producer checkpoint and the first-class raw/native +Opera generation -> complete empty-step initialization -> sample -> persisted +reopen -> combined inventory journey. Require one R01C01 semantic well and +coherent site/channel/Z/time membership and exact source refs. Keep issue 172 +open until that producer acceptance; this source owner takes no science/native +slot. No live-readiness or general biological/calibration correctness claim is +made; issue 335's source equivalence is qualified separately below. +Unrelated existing HTD/grid layout branches are unchanged. The independent +native pixel-size default is removed by the issue 335 addition below. + +Evidence archive: opera_producer_inventory_172_source_evidence.tgz retains +negative/working/final source logs, tiny original-baseline plate, original R0/R1, +complete focused census/AST, import identities, resource summaries, the original +archive census scripts and disposable supervisor/evidence driver. Fresh-extract +byte comparison must pass before cleanup. Own scratch is about 5.2 MiB (far +below 256 MiB). Own clean detached base-proof, R0-tool and NRA-tool worktrees +are disposable and removed after their processes terminate; the source/branch +worktree, original fixtures and all foreign worktrees remain persistent. + +Issue 335: acquisition-owned calibration closure +------------------------------------------------ + +Parent additionally assigned the demonstrated raw/native calibration defect; +read issue 335 body/comments completely and the complete +SYNTHETIC-OPERA-CALIBRATION-20261001.rst parent witness before edits. Existing +live evidence proves the raw installed route only. The same producer previously +wrote Opera XML ImageResolutionX/Y=1.1867525298988041e-6 m while independently +passing 0.65 micrometers to native metadata. This predates this branch. + +BOUND-2 / IDEN-6 / IMPL-1 / TIME-5: traced actual acquisition metadata handlers, +MetadataHandler.source_voxel_spacing and the complete existing SourceVoxelSpacing +physical-unit/scalar/merge contract. The richer authority is the handler's +decoded HTD/XML acquisition calibration, not a new format roster or constant. +Instantiate the already selected handler with a real local disk FileManager. +Native serialization reads that owner's typed spacing once, requires physical +isotropic calibration through its existing contract, and merges it into each +retained typed plane using the existing source-metadata/projection machinery. +SourceProjectionMetadataSerializer publishes those fields and the scalar from +the same typed spacing. The shared algorithm stays on the producer; format +decoding remains on the existing metadata-handler ancestor/leaf contracts. + +Deleted the unconditional 0.65 native argument and the independent public +pixel_size metadata-writer parameter. Migrated the one remaining integration +fixture-helper caller after confirming no active compiler/runtime/renderer PR +owns it. No compatibility argument/alias, silent override, XML input tweak, +calibration lookup map, consumer correction or fallback is added. Raw acquisition +HTD/XML declarations remain unchanged and authoritative. A missing or relative +physical calibration cannot pass SourceVoxelSpacing.require_physical_pixel_size. +The new-capability test now generates its real HTD before the same plane-emission +and calibrated metadata boundary; no fake reader/calibration is supplied. + +Before this addition the new paired control fails in four format/Z subcases: +Opera scalar/XY 0.65 versus 1.1867525298988042; both formats also lacked per-plane +persisted coordinate spacing. Preserve calibration-negative.log. Afterward, +24 combined source-contract methods pass in 2.79 s/94.18 MiB, including the new +paired raw/native ImageXpress and Opera flat/stack test. It proves exact fresh +reader scalar/XY/unit equality, every persisted source projection's typed +spacing, identical physical file membership and TIFF pixel-array parity. No +scientific data or actual installed paired comparison is inferred from that. + +Both issues remain open for parent-owned new-candidate installation, raw/native +initialize/inspect/sample/reopen and combined-inventory acceptance after Hubble's +scientific slot. Original failures, fixtures and frozen H001d installation are +unchanged. PR 333 owns this addition too; no untracked follow-up branch. + +Final calibrated checkpoint and publication boundary +---------------------------------------------------- + +Production calibration qualified at ba41c37e2be5d6e47b59defc334f41dad1b53f0f; +test-only missing-acquisition control added at 67dbefb3c. Twenty-five combined +source-contract methods pass (this task's fourteen plus PR 329's eleven), +5.14 s/94.28 MiB. The additional control removes only its newly generated +disposable XML and proves the existing FileNotFoundError propagates without +publishing invented/default physical calibration. All original/frozen fixtures +are unchanged. Existing 34-method regression shard also passes after calibration, +8.59 s/385.64 MiB. No resource limit is hit. + +Original unchanged calibrated R0 openhcs guard passes against main 2cbfc4a0: +22.20 s/85.63 MiB, exit 0, only GodClassExcess -131; every other delta zero. +Scripts/benchmark trees are byte-identical to the previously guarded merged +checkpoint, so their retained guard inputs/results are unchanged. Calibrated +current-archive census: zero unparsed; string equality -6, optional None checks ++7, long function -1, code lines -40; all other measures zero delta. Additional +AST witnesses enforce calls to source_voxel_spacing, require_physical_pixel_size +and merge_into and deletion of the independent pixel_size parameter. Original +R1 remains unresolved before global analysis; no full NRA/R1 certification or +exception is claimed for this calibrated version. + +Final changed paths: synthetic_data.py; parser_metaprogramming.py; existing +imagexpress.py/opera_phenix.py filename leaves; the one fixture_utils.py caller; +test_synthetic_acquisition_identity.py; this RST and its evidence archive. +No new production nominal class, copied semantic procedure or authority mirror. +The original reciprocal-order cooperative MI controls remain and pass after +the calibration extension through the real acquisition metadata owner. + +Owned disposable worktrees tracked for terminal cleanup: +/home/ts/wt/openhcs-opera-producer-inventory-172-base-proof-20261001, +/home/ts/wt/openhcs-opera-producer-inventory-172-audit-tool-20261001, +/home/ts/wt/openhcs-opera-producer-inventory-172-nra-tool-20261001. +Their sources/status are clean; these contain no initialized child repositories. +The new implementation worktree/branch remains persistent for parent integration. + +Terminal archive verification and cleanup +----------------------------------------- + +Final archive size 1,157,171 bytes; SHA256 +326541e4ccb3a4b5378ad4322614c4d9bdc03448bba9575535548f02285aaa7b. +Fresh extraction to owned persistent +/home/ts/.cache/agent-scratch/opera-producer-inventory-172-archive-check-eFy3aneS +and recursive byte comparison against the original 8.2 MiB scratch PASS. +All supervised commands are terminal, with no resource-limit termination. +Removed only the three verified-clean detached proof/tool worktrees above, +owned original scratch and its verified extraction directory. Evidence is +recoverable from this committed archive; source from its recorded Git objects. +The implementation worktree/branch, parent worktrees, original fixtures, +installed/frozen candidates and scientific slot are unchanged and retained. + +Normal main integration: 8c512d4 +-------------------------------- + +At the parent's explicit request, normally merged current remote main +8c512d404f8707a6a1be311406c0af2a089360f7 into this owned branch. Merge commit +03e3315d03fe775ed57ad71c5805507c946cfcf8 has exactly two parents: the previous +public checkpoint a339a6d20c5c2b8c04ed3fecd3bd1496e7c839cf and that main pin. +No conflict resolution, rebase, force push or authored production change. +The merge was published immediately after the source-contract shard passed. + +Re-read the full current nra-refactoring SKILL.md and installed refactor-audit +SKILL.md, then the authoritative refactor-audit.skill ZIP entrypoint, pattern +README, surface receipt and complete identity, boundaries, membership, +over-time and implementation references before integration decisions. +ZIP SHA256 remains +100fbe8ef89664b866777e87b2c8640a3432e8a10e9188dff81c97942d551bf6. +Considered IDEN-3/6, BOUND-2, IMPL-1/5/12, MEMB-1/4/5 and TIME-3/5/9 against +the existing witnesses above. No new semantic owner or declaration surface is +needed for this integration. Both reciprocal-order, genuinely new capability +diamond controls execute through unchanged production consumers and still pass. +The existing ancestor algorithm, declaration hooks and automatic registration +remain exact bytes; no copied implementation, fake subtype or mirrored roster. + +Main contributes 32 files (737 insertions/99 deletions), principally the +already-owned compiler/output/materialization qualification. These are upstream +integration, not a competing implementation or edits to another owner's files. +Reviewed the adjacent virtual_workspace_metadata and artifacts changes: durable +subdirectory document loading and measurement row-domain validation respectively. +None of the four assigned production modules, fixture-helper caller, assigned +test or frozen PR 329 XML parser changes from a339. Ordinary public imports +prove the four assigned modules resolve to this worktree and have the exact old +Git bytes; the adjacent metadata/artifact modules have the exact new main bytes. +The recorded external/PolyStore gitlink is +84f322e46871de5ed47e7fd20976ad03e533c1c4, confirmed by git ls-tree HEAD. +The read-only source-check dependency is the historical qualified source +91fd7e854ca760a16e7472d8d558b4012e89b6ba, with identical production source/tests +to that merged PolyStore pin. No dependency or installed package was modified. + +Bounded integration evidence (same existing Python 3.12, one CPU, single-thread +math libraries, bytecode disabled, supervised process-group RSS/time/scratch): + +* Public source-contract discovery: 25 methods PASS, 2.84 s, 94.39 MiB peak. +* Existing zero-geometry/randomness/ImageXpress Bio-Formats/source-projection + regression shard: 34 tests PASS, 7.99 s, 388.49 MiB peak. +* Exact merge/source/import/archive identity control: PASS, 1.42 s, 67.92 MiB + peak; its log records actual paths, hashes and Git authority pins. + +No shard exceeds 512 MiB/60 s or 256 MiB scratch. Resource guard before testing +returned warning exit 2: 21.0 GiB available RAM, home 19.9 GiB below the user's +warning-only 20 GiB threshold, historical swap 13.7 GiB, root 8.2 GiB. This is +not recorded as a guard pass; bounded source-only work fits the actual resources. +Commands/results and copied parent receipt are retained separately in +opera_producer_inventory_172_main8c512_evidence.tgz. The original archive above +is not regenerated: size/hash and every byte remain identical to a339, with its +exact historical source/baseline pins. Its old R0/census evidence is not relabeled +as a guard run on the later foreign main delta. No metric exception or gaming. + +Latest full-context R1 limit supersedes, without deleting, the historical local +uninitialized-submodule failure above. Parent materialized all eight recorded +gitlinks plus pinned NRA 0844525. Baseline completed 3068 projections; the +original supervised group hit the enforced 512 MiB cap at 83.29 s/530.62 MiB +peak before head comparison (original 160 s budget). R1 is INCOMPLETE, neither +passed nor waived. No changed roots/budget/cap, substitute scan or full NRA claim. + +Read the entire parent producer-installed-20261001/ACCEPTANCE.rst. Parent verified +30 original installed MCP responses, exit 0, at source +67dbefb3caf8848901f6561d6bd274a84accee92 (a339 adds only docs/archive): four tiny +raw/native fixtures, four complete typed addresses each, physical calibration +parity, four pairs of equal 64-pixel samples and four persisted inventories. +This is parent-owned installed evidence, not a repeated run by this source owner. +It does not automatically qualify the later main compiler/runtime integration. +Parent checks source delta and qualification applicability and owns merge/issue +disposition. Issues 172/335 are not closed by this receipt. Separate initial +read-only ImageXpress Z-folder inspection defect 337 belongs to Arendt; no fix +or overlapping ImageXpress edit is included here. No native/MCP/science launch, +lock use, installation, frozen candidate or original fixture mutation. + +New owned disposable scratch: +/home/ts/.cache/agent-scratch/opera-producer-inventory-172-main8c512-20261001. +Archive verification and removal occur only after every supervised process is +terminal. Source worktree stays persistent; historical archives remain intact. + +Fresh integration archive verification: 348,147 bytes, SHA256 +df6cb051e96b6eded6860958ffecda1e3debe57412342676becffb9ba0676c4a. +Extracted into owned persistent +/home/ts/.cache/agent-scratch/opera-producer-inventory-172-main8c512-verify-eGIHgycX; +recursive byte comparison against the original 631,827-byte scratch PASS. +The copied parent ACCEPTANCE.rst has SHA256 +05f71852e64bbe3e47508c18abd48b0cf3564e55577cda156cf753e99706dd5e. +All three supervised shards are terminal with exit 0 and limit null. Remove +only this run's original scratch, verified extraction and temporary PR-body +file after publication; evidence remains recoverable from the committed archive +and public PR. No historical evidence or persistent implementation tree removed. diff --git a/docs/validation/opera_producer_inventory_172_main8c512_evidence.tgz b/docs/validation/opera_producer_inventory_172_main8c512_evidence.tgz new file mode 100644 index 0000000000..3714662c0f Binary files /dev/null and b/docs/validation/opera_producer_inventory_172_main8c512_evidence.tgz differ diff --git a/docs/validation/opera_producer_inventory_172_source_evidence.tgz b/docs/validation/opera_producer_inventory_172_source_evidence.tgz new file mode 100644 index 0000000000..d3d8dbed17 Binary files /dev/null and b/docs/validation/opera_producer_inventory_172_source_evidence.tgz differ diff --git a/openhcs/core/components/parser_metaprogramming.py b/openhcs/core/components/parser_metaprogramming.py index 8589e42041..e16f517a47 100644 --- a/openhcs/core/components/parser_metaprogramming.py +++ b/openhcs/core/components/parser_metaprogramming.py @@ -202,6 +202,22 @@ def construct_filename(self, components: FilenameParseResult) -> str: """Construct a filename from one nominal component result.""" pass + def construct_acquisition_filename( + self, + components: FilenameParseResult, + *, + include_all_components: bool = False, + plate_name: str | None = None, + include_site: bool = True, + include_channel: bool = True, + ) -> str: + """Spell a physical acquisition name; semantic identity stays complete. + + Formats with folder-encoded axes or acquisition-specific spelling own + those conventions in their existing parser, not in the producer. + """ + return self.construct_filename(components) + def get_component_names(self) -> tuple[str, ...]: """Get all component names for this parser.""" return AllComponents.ordered_names() diff --git a/openhcs/demo/synthetic_data.py b/openhcs/demo/synthetic_data.py index 8a277bb2ec..a98758b35c 100644 --- a/openhcs/demo/synthetic_data.py +++ b/openhcs/demo/synthetic_data.py @@ -24,8 +24,8 @@ generator.generate_dataset() """ -import json from datetime import datetime +from dataclasses import replace from pathlib import Path from typing import Optional, Tuple, List, Dict, Any @@ -34,9 +34,16 @@ from skimage import draw, filters from openhcs.constants.constants import AllComponents -from openhcs.microscopes.imagexpress import ImageXpressFilenameParser -from openhcs.microscopes.opera_phenix import OperaPhenixFilenameParser +from openhcs.core.source_projection import ( + OpenHCSPlaneAddress, SourcePlaneProjection, SourceProjectionSet, + SourceProjectionMetadataSerializer, +) +from openhcs.core.virtual_workspace_metadata import AtomicMetadataWriter, get_metadata_path +from openhcs.core.source_metadata import SourceVoxelSpacing, source_metadata_dict +from openhcs.microscopes.microscope_base import MICROSCOPE_HANDLERS from polystore.constants import Backend +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager from polystore.virtual_workspace import SourcePixelRef @@ -151,9 +158,19 @@ def __init__( self.imagexpress_bioformats_compatible = imagexpress_bioformats_compatible self.openhcs_format = openhcs_format - # Create parser instances for filename construction - self.imagexpress_parser = ImageXpressFilenameParser() - self.operaphenix_parser = OperaPhenixFilenameParser() + # Decode the external format once through the registered declaration. + handler_type = next( + (owner for owner in MICROSCOPE_HANDLERS.values() + if owner.__name__ == f"{format}Handler"), + None, + ) + if handler_type is None: + raise ValueError(f"No declared microscope handler for format {format!r}") + self.microscope_handler = handler_type( + FileManager({Backend.DISK.value: DiskStorageBackend()}) + ) + self.parser = self.microscope_handler.parser + self._source_planes: list[SourcePlaneProjection] = [] # Always auto-calculate image size from grid and tile parameters self.image_size = self._calculate_image_size( @@ -609,9 +626,10 @@ def generate_opera_phenix_index_xml(self, plate_name): # Convert well names like 'A01' to row and column indices well_indices = [] for well in self.wells: - row = ord(well[0]) - ord("A") + 1 # A -> 1, B -> 2, etc. - col = int(well[1:3]) - well_indices.append((row, col)) + row_name, column_name = self.parser.extract_component_coordinates(well) + row = ord(row_name) - ord("A") + 1 + col = int(column_name) + well_indices.append((well, row, col)) # Calculate pixel size in meters (for ImageResolutionX/Y) # Default is 0.65 µm, but we'll use a more realistic value for Opera Phenix @@ -636,13 +654,13 @@ def generate_opera_phenix_index_xml(self, plate_name): 12""" # Add wells - for row, col in well_indices: + for well, row, col in well_indices: xml_content += f'\n ' xml_content += "\n \n \n " # Add well details - for row, col in well_indices: + for well, row, col in well_indices: xml_content += f"\n \n {row:02d}{col:02d}\n {row}\n {col}" # Add images for each site and channel @@ -731,7 +749,7 @@ def generate_opera_phenix_index_xml(self, plate_name): ) # Add detailed image information for each image - for row, col in well_indices: + for well, row, col in well_indices: for site in range(1, self.grid_size[0] * self.grid_size[1] + 1): # Get grid position for this field ID using Opera Phenix pattern site_row, site_col = field_pattern[site] @@ -758,7 +776,9 @@ def generate_opera_phenix_index_xml(self, plate_name): image_id = f"{row:02d}{col:02d}K1F{site}P{z}R{channel}" # Create URL (filename) - url = f"r{row:02d}c{col:02d}f{site}p{z:02d}-ch{channel}sk1fk1fl1.tiff" + url = self.parser.construct_acquisition_filename( + self._plane_components(well, site, channel, z) + ) # Add image element xml_content += f""" @@ -838,6 +858,7 @@ def _site_position(self, row: int, col: int) -> tuple[int, int]: def generate_dataset(self): """Generate the complete dataset.""" + self._source_planes = [] print(f"Generating synthetic microscopy dataset in {self.output_dir}") print(f"Grid size: {self.grid_size[0]}x{self.grid_size[1]}") print(f"Wavelengths: {self.wavelengths}") @@ -933,38 +954,9 @@ def generate_dataset(self): x_pos : x_pos + self.tile_size[0], ] - # Create filename based on format - if self.format == "ImageXpress": - filename = self.imagexpress_filename( - well=well, - site=site_index, - channel=wavelength, - z_index=z_level, - ) - else: # OperaPhenix - # Opera Phenix format: rXXcYYfZZZpWW-chVskNfkNflN.tiff - # Extract row and column from well ID (e.g., 'A01' -> row=1, col=1) - row = ord(well[0]) - ord("A") + 1 - col = int(well[1:3]) - filename = f"r{row:02d}c{col:02d}f{site_index}p{z_level:02d}-ch{wavelength}sk1fk1fl1.tiff" - filepath = target_dir / filename - - # Skip this file if it's in the skip list - if filename in self.skip_files: - print( - f" Skipped tile: {filename} (simulating missing image)" - ) - site_index += 1 - continue - - # Save image without compression - tifffile.imwrite(filepath, tile, compression=None) - - # Print progress with full path for debugging - print( - f" Saved tile: {target_dir.name}/{filename} (position: {x_pos}, {y_pos})" + self._write_plane( + tile, target_dir, well, site_index, wavelength, z_level ) - print(f" Full path: {filepath.resolve()}") site_index += 1 else: # For single Z level (no Z-stack), just save files directly in TimePoint_1 @@ -990,36 +982,8 @@ def generate_dataset(self): x_pos : x_pos + self.tile_size[0], ] - # Create filename based on format - if self.format == "ImageXpress": - filename = self.imagexpress_filename( - well=well, - site=site_index, - channel=wavelength, - z_index=1, - ) - else: # OperaPhenix - # Opera Phenix format: rXXcYYfZZZpWW-chVskNfkNflN.tiff - # Extract row and column from well ID (e.g., 'A01' -> row=1, col=1) - row = ord(well[0]) - ord("A") + 1 - col = int(well[1:3]) - filename = f"r{row:02d}c{col:02d}f{site_index}p01-ch{wavelength}sk1fk1fl1.tiff" - filepath = self.timepoint_dir / filename - - # Skip this file if it's in the skip list - if filename in self.skip_files: - print( - f" Skipped tile: {filename} (simulating missing image)" - ) - site_index += 1 - continue - - # Save image without compression - tifffile.imwrite(filepath, tile, compression=None) - - # Print progress - print( - f" Saved tile: {filename} (position: {x_pos}, {y_pos})" + self._write_plane( + tile, self.timepoint_dir, well, site_index, wavelength, 1 ) site_index += 1 @@ -1027,53 +991,45 @@ def generate_dataset(self): # If OpenHCS format requested, generate metadata file if self.openhcs_format: - # Determine subdirectory based on format - if self.format == "ImageXpress": - sub_dir = "TimePoint_1" - else: # OperaPhenix - sub_dir = "Images" - - self.generate_openhcs_metadata(sub_dir=sub_dir, pixel_size=0.65) - - def imagexpress_filename( - self, - *, - well: str, - site: int, - channel: int, - z_index: int, - ) -> str: - """Return an ImageXpress synthetic filename for the configured compatibility mode.""" - if self.imagexpress_bioformats_compatible: - plate_name = self.output_dir.name - parts = [f"{plate_name}_{well}"] - if self.grid_size[0] * self.grid_size[1] > 1: - parts.append(f"_s{site}") - if self.wavelengths > 1: - parts.append(f"_w{channel}") - return "".join(parts) + ".tif" - return self.imagexpress_parser.construct_filename( - self.imagexpress_parser.bind_declared_values( - ( - (AllComponents.WELL, well), - (AllComponents.SITE, site), - (AllComponents.CHANNEL, channel), - ( - AllComponents.Z_INDEX, - z_index if self.include_all_components else None, - ), - ( - AllComponents.TIMEPOINT, - 1 if self.include_all_components else None, - ), - ), - extension=".tif", - ) + self.generate_openhcs_metadata(sub_dir=self.timepoint_dir.name) + + def _plane_components(self, well: str, site: int, channel: int, z_index: int): + """Bind the complete acquisition address before physical spelling.""" + return self.parser.bind_declared_values( + ((AllComponents.WELL, well), (AllComponents.SITE, site), + (AllComponents.CHANNEL, channel), (AllComponents.Z_INDEX, z_index), + (AllComponents.TIMEPOINT, 1)) ) - def generate_openhcs_metadata( - self, sub_dir: str = "images", pixel_size: float = 0.65 - ): + def _write_plane(self, tile, target_dir: Path, well: str, site: int, + channel: int, z_index: int) -> None: + components = self._plane_components(well, site, channel, z_index) + # The parser owns normalization (e.g. A01 input -> Opera R01C01). + canonical_name = self.parser.construct_filename(components) + parsed = self.parser.parse_filename(canonical_name) + if parsed is None: + raise ValueError(f"Declared parser cannot reopen {canonical_name!r}") + address = OpenHCSPlaneAddress.from_component_values(parsed.declared_values()) + filename = self.parser.construct_acquisition_filename( + components, + include_all_components=self.include_all_components, + plate_name=self.output_dir.name if self.imagexpress_bioformats_compatible else None, + include_site=self.grid_size[0] * self.grid_size[1] > 1, + include_channel=self.wavelengths > 1, + ) + if filename in self.skip_files: + print(f" Skipped tile: {filename} (simulating missing image)") + return + filepath = target_dir / filename + tifffile.imwrite(filepath, tile, compression=None) + self._source_planes.append(SourcePlaneProjection( + address=address, + ref=SourcePixelRef(backend=Backend.DISK.value, + backend_address=filepath.relative_to(self.output_dir).as_posix()), + )) + print(f" Saved tile: {filepath}") + + def generate_openhcs_metadata(self, sub_dir: str = "images"): """ Generate OpenHCS metadata file for the synthetic dataset. @@ -1082,116 +1038,36 @@ def generate_openhcs_metadata( Args: sub_dir: Subdirectory name where images are located (default: "images") - pixel_size: Pixel size in microns (default: 0.65) + + Calibration is decoded by the declared acquisition metadata owner, not + supplied independently of the generated HTD/XML. """ print("\nGenerating OpenHCS metadata...") - # Collect all image files - image_files = [] - workspace_mapping: Dict[str, SourcePixelRef] = {} - if self.format == "ImageXpress": - # ImageXpress stores images in TimePoint_1/ZStep_X/ or TimePoint_1/ for flat plates - timepoint_dir = self.output_dir / "TimePoint_1" - if self.z_stack_levels > 1: - # Z-stack: images in ZStep folders - for z in range(1, self.z_stack_levels + 1): - zstep_dir = timepoint_dir / f"ZStep_{z}" - if zstep_dir.exists(): - for img_file in sorted(zstep_dir.glob("*.tif")): - virtual_path = f"{sub_dir}/{img_file.name}" - image_files.append(virtual_path) - workspace_mapping[virtual_path] = SourcePixelRef( - backend=Backend.DISK.value, - backend_address=img_file.relative_to( - self.output_dir - ).as_posix(), - ) - else: - # Flat: images directly in TimePoint_1 - if timepoint_dir.exists(): - for img_file in sorted(timepoint_dir.glob("*.tif")): - rel_path = f"{sub_dir}/{img_file.name}" - image_files.append(rel_path) - else: # OperaPhenix - # OperaPhenix stores images in Images/ folder - images_dir = self.output_dir / "Images" - if images_dir.exists(): - for img_file in sorted(images_dir.glob("*.tiff")): - rel_path = f"{sub_dir}/{img_file.name}" - image_files.append(rel_path) - - # Build component metadata dictionaries - channels = {str(i): str(i) for i in range(1, self.wavelengths + 1)} - wells = {well: well for well in self.wells} - - # Calculate number of sites from grid - num_sites = self.grid_size[0] * self.grid_size[1] - sites = {str(i): str(i) for i in range(1, num_sites + 1)} - - # Z-indexes - z_indexes = ( - {str(i): str(i) for i in range(1, self.z_stack_levels + 1)} - if self.z_stack_levels > 1 - else None - ) - - # Timepoints (always 1 for synthetic data) - timepoints = {"1": "1"} - - # Determine parser name based on format - parser_name = ( - "ImageXpressFilenameParser" - if self.format == "ImageXpress" - else "OperaPhenixFilenameParser" + # Serialize one typed authority: only planes actually written by this run. + spacing = self.microscope_handler.metadata_handler.source_voxel_spacing(self.output_dir) + pixel_size = SourceVoxelSpacing.require_physical_pixel_size((spacing,)) + calibrated_planes = [] + for plane in self._source_planes: + source_metadata = source_metadata_dict(plane.source_metadata) + spacing.merge_into(source_metadata, path=plane.ref.backend_address) + calibrated_planes.append(replace(plane, source_metadata=source_metadata)) + projections = SourceProjectionSet(tuple(calibrated_planes)) + metadata = SourceProjectionMetadataSerializer( + self.parser, image_extension=self.parser.DEFAULT_EXTENSION, path_prefix=sub_dir + ).metadata_dict( + projections, + microscope_handler_name=self.microscope_handler.microscope_type, + source_filename_parser_name=type(self.parser).__name__, + grid_dimensions=list(self.grid_size), + pixel_size=pixel_size, + available_backends={Backend.DISK.value: True, Backend.VIRTUAL_WORKSPACE.value: True}, + main=True, ) - microscope_handler = ( - "imagexpress" if self.format == "ImageXpress" else "operaphenix" - ) - - # Build metadata structure - metadata = { - "subdirectories": { - sub_dir: { - "microscope_handler_name": microscope_handler, - "source_filename_parser_name": parser_name, - "grid_dimensions": list(self.grid_size), - "pixel_size": pixel_size, - "image_files": image_files, - "channels": channels, - "wells": wells, - "sites": sites, - "z_indexes": z_indexes, - "timepoints": timepoints, - "available_backends": { - "disk": True, - **({"virtual_workspace": True} if workspace_mapping else {}), - }, - **( - { - "workspace_mapping": { - virtual_path: source_ref.to_workspace_mapping() - for virtual_path, source_ref in workspace_mapping.items() - } - } - if workspace_mapping - else {} - ), - "main": True, - } - } - } - - # Write metadata file - metadata_path = self.output_dir / "openhcs_metadata.json" - with open(metadata_path, "w") as f: - json.dump(metadata, f, indent=2) + metadata_path = get_metadata_path(self.output_dir) + AtomicMetadataWriter().replace_subdirectory_metadata(metadata_path, sub_dir, metadata) print(f"✓ Generated OpenHCS metadata: {metadata_path}") - print(f" - {len(image_files)} image files") - print(f" - {len(wells)} wells") - print(f" - {len(channels)} channels") - print(f" - {len(sites)} sites") - if z_indexes: - print(f" - {len(z_indexes)} z-levels") + print(f" - {len(projections.projections)} source planes") return metadata_path diff --git a/openhcs/microscopes/imagexpress.py b/openhcs/microscopes/imagexpress.py index 60a8c20946..6d70a28ef5 100644 --- a/openhcs/microscopes/imagexpress.py +++ b/openhcs/microscopes/imagexpress.py @@ -416,6 +416,10 @@ def construct_filename( site_padding: int = 3, z_padding: int = 3, timepoint_padding: int = 3, + *, + plate_name: str | None = None, + include_site: bool = True, + include_channel: bool = True, ) -> str: """Construct an ImageXpress filename from nominal component values.""" @@ -425,13 +429,13 @@ def construct_filename( z_index = components.value_for(AllComponents.Z_INDEX) timepoint = components.value_for(AllComponents.TIMEPOINT) - parts = [well] + parts = [f"{plate_name}_{well}" if plate_name is not None else well] - # Always add site - parts.append(f"_s{format_filename_component(site, site_padding)}") + if include_site: + parts.append(f"_s{format_filename_component(site, site_padding)}") - # Always add channel - parts.append(f"_w{format_filename_component(channel)}") + if include_channel: + parts.append(f"_w{format_filename_component(channel)}") if z_index is not None: parts.append(f"_z{format_filename_component(z_index, z_padding)}") @@ -442,6 +446,28 @@ def construct_filename( base_name = "".join(parts) return f"{base_name}{components.extension}" + def construct_acquisition_filename( + self, + components: FilenameParseResult, + *, + include_all_components: bool = False, + plate_name: str | None = None, + include_site: bool = True, + include_channel: bool = True, + ) -> str: + """Own MetaXpress acquisition spelling, including folder-only Z axes.""" + if not include_all_components or plate_name is not None: + components = components.with_values( + ((AllComponents.Z_INDEX, None), (AllComponents.TIMEPOINT, None)) + ) + return self.construct_filename( + components, + site_padding=0 if plate_name is not None else 3, + plate_name=plate_name, + include_site=include_site if plate_name is not None else True, + include_channel=include_channel if plate_name is not None else True, + ) + class ImageXpressMetadataHandler(DiskImageFileListingMetadataHandler): """ diff --git a/openhcs/microscopes/opera_phenix.py b/openhcs/microscopes/opera_phenix.py index 2b14efe966..5ae088b1ec 100644 --- a/openhcs/microscopes/opera_phenix.py +++ b/openhcs/microscopes/opera_phenix.py @@ -522,15 +522,9 @@ def construct_filename( z_index = components.required_value(AllComponents.Z_INDEX) timepoint = components.required_value(AllComponents.TIMEPOINT) - # Extract row and column from well name - # Check if well is in Opera Phenix format (e.g., 'R01C03') - match = self._well_pattern.match(well) - if match: - # Extract row and column from Opera Phenix format - row = int(match.group(1)) - col = int(match.group(2)) - else: - raise ValueError(f"Invalid well format: {well}. Expected format: 'R01C03'") + row_name, column_name = self.extract_component_coordinates(well) + row = ord(row_name) - ord("A") + 1 + col = int(column_name) # Construct filename in Opera Phenix format site_part = f"f{format_filename_component(site, site_padding)}" @@ -542,6 +536,18 @@ def construct_filename( f"-ch{format_filename_component(channel)}{sk_part}fk1fl1{components.extension}" ) + def construct_acquisition_filename( + self, + components: FilenameParseResult, + *, + include_all_components: bool = False, + plate_name: str | None = None, + include_site: bool = True, + include_channel: bool = True, + ) -> str: + """Keep Harmony's physical field/plane spelling on the format owner.""" + return self.construct_filename(components, site_padding=0, z_padding=2) + def remap_field_in_filename( self, filename: str, xml_parser: Optional[OperaPhenixXmlParser] = None ) -> str: diff --git a/tests/integration/helpers/fixture_utils.py b/tests/integration/helpers/fixture_utils.py index 21a31b444a..c720deccb5 100644 --- a/tests/integration/helpers/fixture_utils.py +++ b/tests/integration/helpers/fixture_utils.py @@ -355,7 +355,7 @@ def create_synthetic_plate_data( else: # OperaPhenix sub_dir = "Images" - generator.generate_openhcs_metadata(sub_dir=sub_dir, pixel_size=0.65) + generator.generate_openhcs_metadata(sub_dir=sub_dir) # Return the plate directory return plate_dir diff --git a/tests/source_contracts/test_synthetic_acquisition_identity.py b/tests/source_contracts/test_synthetic_acquisition_identity.py new file mode 100644 index 0000000000..48597aae95 --- /dev/null +++ b/tests/source_contracts/test_synthetic_acquisition_identity.py @@ -0,0 +1,323 @@ +"""Tiny producer/metadata controls using real public source imports, no MCP/JVM.""" +from contextlib import redirect_stdout +from io import StringIO +import json +from pathlib import Path +import tempfile +import unittest +import xml.etree.ElementTree as ET + +import numpy as np +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +import openhcs.demo.synthetic_data as producer_module +from openhcs.constants.constants import AllComponents +from openhcs.core.source_projection import ( + OpenHCSPlaneAddress, SourcePlaneProjection, SourceProjectionSet, + SourceProjectionMetadataSerializer, +) +from openhcs.core.components.parser_metaprogramming import GenericFilenameParser +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.virtual_workspace_metadata import ( + FIELDS, VirtualWorkspaceSourceProjectionEntries, component_metadata_field, + AtomicMetadataWriter, get_metadata_path, +) +from openhcs.demo.synthetic_data import SyntheticMicroscopyGenerator +from openhcs.microscopes.microscope_interfaces import FilenameParser +from openhcs.microscopes.openhcs import OpenHCSMetadataHandler +from openhcs.microscopes.imagexpress import ImageXpressFilenameParser, ImageXpressHandler +from openhcs.microscopes.microscope_base import MICROSCOPE_HANDLERS +from polystore.virtual_workspace import SourcePixelRef + + +class SyntheticAcquisitionIdentity(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="synthetic-identity-172-") + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + + def generate(self, format, *, z_levels=1, explicit=False, bioformats=False, + native=True, wells=("A01",), skip_files=(), grid=(1, 1), channels=2): + plate = self.root / str(len(tuple(self.root.iterdir()))) + with redirect_stdout(StringIO()): + generator = SyntheticMicroscopyGenerator( + output_dir=str(plate), format=format, grid_size=grid, + tile_size=(32, 32), overlap_percent=0, stage_error_px=0, + wavelengths=channels, z_stack_levels=z_levels, num_cells=4, + wells=list(wells), random_seed=7, include_all_components=explicit, + imagexpress_bioformats_compatible=bioformats, + openhcs_format=native, skip_files=list(skip_files), + ) + generator.generate_dataset() + return generator, plate + + def document(self, plate, subdirectory=None): + document = json.loads(get_metadata_path(plate).read_text()) + entries = document[FIELDS.SUBDIRECTORIES] + return entries[subdirectory] if subdirectory else next(iter(entries.values())) + + def assert_coherent(self, plate, metadata, count): + paths = metadata[FIELDS.IMAGE_FILES] + self.assertEqual(len(paths), count) + self.assertEqual(len(set(paths)), count) + projections = VirtualWorkspaceSourceProjectionEntries.from_subdirectory(metadata).entries + self.assertEqual(set(projections), set(paths)) + self.assertEqual(set(metadata[FIELDS.WORKSPACE_MAPPING]), set(paths)) + parser = FilenameParser.__registry__[metadata[FIELDS.SOURCE_FILENAME_PARSER_NAME]]() + address_sets = {component: set() for component in AllComponents} + physical_refs = set() + for path, projection in projections.items(): + parsed = parser.parse_filename(Path(path).name) + self.assertIsNotNone(parsed) + self.assertEqual(OpenHCSPlaneAddress.from_component_values(parsed.declared_values()), projection.address) + ref = SourcePixelRef.from_workspace_mapping(metadata[FIELDS.WORKSPACE_MAPPING][path]) + self.assertEqual(ref, projection.ref) + self.assertEqual(ref.backend, "disk") + physical = plate / ref.backend_address + self.assertTrue(physical.is_file()) + self.assertEqual(tifffile.imread(physical).shape, (32, 32)) + physical_refs.add(ref.backend_address) + for component in AllComponents: + address_sets[component].add(projection.address.value_for(component)) + self.assertEqual(len(physical_refs), count) + for component in AllComponents: + self.assertEqual(set(metadata[component_metadata_field(component)]), address_sets[component]) + # A fresh real metadata handler reads the persisted component authority. + reopened = OpenHCSMetadataHandler( + FileManager({"disk": DiskStorageBackend()}) + ).component_value_set(plate) + for component in AllComponents: + self.assertEqual(set(reopened.values_for(component)), address_sets[component]) + return address_sets + + def test_source_identity(self): + expected = Path(__file__).resolve().parents[2] / "openhcs/demo/synthetic_data.py" + self.assertEqual(Path(producer_module.__file__).resolve(), expected) + + def test_raw_and_native_family_all_components_and_exact_source_refs(self): + for format in ("ImageXpress", "OperaPhenix"): + for z_levels in (1, 2): + for explicit in (False, True): + for native in (False, True): + with self.subTest(format=format, z_levels=z_levels, explicit=explicit, native=native): + generator, plate = self.generate(format, z_levels=z_levels, + explicit=explicit, native=native, wells=("A01", "D12"), grid=(1, 2)) + if not native: + self.assertFalse(get_metadata_path(plate).exists()) + with redirect_stdout(StringIO()): + generator.generate_openhcs_metadata(sub_dir="custom") + metadata = self.document(plate) + values = self.assert_coherent(plate, metadata, 8 * z_levels) + self.assertEqual(values[AllComponents.WELL], + {"A01", "D12"} if format == "ImageXpress" else {"R01C01", "R04C12"}) + self.assertEqual(values[AllComponents.SITE], {"1", "2"}) + self.assertEqual(values[AllComponents.CHANNEL], {"1", "2"}) + self.assertEqual(values[AllComponents.Z_INDEX], {str(z) for z in range(1, z_levels + 1)}) + self.assertEqual(values[AllComponents.TIMEPOINT], {"1"}) + self.assertEqual(metadata[FIELDS.MICROSCOPE_HANDLER_NAME], + "imagexpress" if format == "ImageXpress" else "opera_phenix") + + def test_opera_original_well_disagreement_is_not_a_second_inventory_key(self): + generator, plate = self.generate("OperaPhenix", explicit=True) + metadata = self.document(plate) + parser = FilenameParser.__registry__[metadata[FIELDS.SOURCE_FILENAME_PARSER_NAME]]() + parsed_wells = {parser.parse_filename(path.name).value_for(AllComponents.WELL) + for path in plate.rglob("*.tiff")} + self.assertEqual(set(metadata[FIELDS.WELLS]), parsed_wells) + self.assertEqual(set(metadata[FIELDS.WELLS]) | parsed_wells, {"R01C01"}) + self.assert_coherent(plate, metadata, 2) + + def test_imagexpress_bioformats_stack_retains_folder_axis_and_refs(self): + generator, plate = self.generate("ImageXpress", z_levels=2, bioformats=True, + wells=("A01",), grid=(1, 2)) + metadata = self.document(plate) + self.assert_coherent(plate, metadata, 8) + physical = {projection.ref.backend_address for projection in + VirtualWorkspaceSourceProjectionEntries.from_subdirectory(metadata).entries.values()} + self.assertEqual({Path(path).parent.name for path in physical}, {"ZStep_1", "ZStep_2"}) + + def test_skipped_plane_cannot_leave_phantom_catalog_or_reference(self): + generator, plate = self.generate("OperaPhenix", skip_files=("r01c01f1p01-ch2sk1fk1fl1.tiff",)) + metadata = self.document(plate) + values = self.assert_coherent(plate, metadata, 1) + self.assertEqual(values[AllComponents.CHANNEL], {"1"}) + + def test_repeat_generation_replaces_emitted_plane_inventory(self): + generator, plate = self.generate("OperaPhenix", z_levels=2) + with redirect_stdout(StringIO()): + generator.generate_dataset() + self.assert_coherent(plate, self.document(plate), 4) + + def test_sparse_sites_are_derived_from_saved_planes(self): + skipped = tuple(f"r01c01f{site}p01-ch{channel}sk1fk1fl1.tiff" + for site in (1, 3) for channel in (1, 2)) + generator, plate = self.generate("OperaPhenix", grid=(1, 4), skip_files=skipped) + values = self.assert_coherent(plate, self.document(plate), 4) + self.assertEqual(values[AllComponents.SITE], {"2", "4"}) + + def test_new_complete_address_needs_no_metadata_variant(self): + generator, plate = self.generate("OperaPhenix") + with redirect_stdout(StringIO()): + generator._write_plane(np.full((32, 32), 172, dtype=np.uint16), + plate / "Images", "D12", 17, 7, 99) + generator.generate_openhcs_metadata(sub_dir="Images") + values = self.assert_coherent(plate, self.document(plate), 3) + self.assertEqual(values[AllComponents.WELL], {"R01C01", "R04C12"}) + self.assertEqual(values[AllComponents.Z_INDEX], {"1", "99"}) + self.assertEqual(values[AllComponents.SITE], {"1", "17"}) + self.assertEqual(values[AllComponents.CHANNEL], {"1", "2", "7"}) + np.testing.assert_array_equal(tifffile.imread(plate / "Images/r04c12f17p99-ch7sk1fk1fl1.tiff"), + np.full((32, 32), 172, dtype=np.uint16)) + + def test_bioformats_scalar_tokens_can_be_absent_without_losing_identity(self): + generator, plate = self.generate("ImageXpress", channels=1, bioformats=True) + self.assertEqual({path.name for path in plate.rglob("*.tif")}, {f"{plate.name}_A01.tif"}) + values = self.assert_coherent(plate, self.document(plate), 1) + for component in AllComponents: + if component is not AllComponents.WELL: + self.assertEqual(values[component], {"1"}) + + def test_xml_urls_and_physical_names_share_parser_owner(self): + generator, plate = self.generate("OperaPhenix", z_levels=2, + wells=("A01", "D12"), grid=(2, 2)) + root = ET.parse(plate / "Images/Index.xml").getroot() + urls = {element.text for element in root.iter() if element.tag.endswith("}URL")} + self.assertEqual(urls, {path.name for path in plate.rglob("*.tiff")}) + self.assert_coherent(plate, self.document(plate), 32) + + def test_nominal_acquisition_operation_respects_existing_mro(self): + generator, plate = self.generate("OperaPhenix") + parser = generator.parser + self.assertIsInstance(parser, GenericFilenameParser) + components = generator._plane_components("D12", 17, 7, 99) + canonical = GenericFilenameParser.construct_acquisition_filename(parser, components) + self.assertEqual(canonical, parser.construct_filename(components)) + self.assertEqual(parser.construct_acquisition_filename(components), + "r04c12f17p99-ch7sk1fk1fl1.tiff") + + def test_raw_native_acquisition_calibration_and_source_units_are_identical(self): + for format in ("ImageXpress", "OperaPhenix"): + for z_levels in (1, 2): + with self.subTest(format=format, z_levels=z_levels): + raw_generator, raw_plate = self.generate(format, native=False, z_levels=z_levels) + native_generator, native_plate = self.generate(format, native=True, z_levels=z_levels) + acquisition = type(raw_generator.microscope_handler)( + FileManager({"disk": DiskStorageBackend()})).metadata_handler + raw_spacing = acquisition.source_voxel_spacing(raw_plate) + native_reader = OpenHCSMetadataHandler(FileManager({"disk": DiskStorageBackend()})) + self.assertEqual(native_reader.source_voxel_spacing(native_plate), raw_spacing) + self.assertEqual(native_reader.get_pixel_size(native_plate), acquisition.get_pixel_size(raw_plate)) + projections = VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + self.document(native_plate)).entries + for projection in projections.values(): + self.assertEqual(SourceVoxelSpacing.from_source_metadata(projection.source_metadata), raw_spacing) + raw_files = {path.relative_to(raw_plate) for path in raw_plate.rglob("*.tif*")} + native_files = {path.relative_to(native_plate) for path in native_plate.rglob("*.tif*")} + self.assertEqual(raw_files, native_files) + for path in raw_files: + np.testing.assert_array_equal(tifffile.imread(raw_plate / path), + tifffile.imread(native_plate / path)) + + def test_absent_acquisition_cannot_publish_default_physical_calibration(self): + generator, plate = self.generate("OperaPhenix", native=False) + # Only this new disposable control's XML is removed. Historical and + # frozen fixtures are not inputs to this negative admission test. + (plate / "Images/Index.xml").unlink() + with self.assertRaises(FileNotFoundError), redirect_stdout(StringIO()): + generator.generate_openhcs_metadata(sub_dir="Images") + self.assertFalse(get_metadata_path(plate).exists()) + + def test_new_nominal_capabilities_cooperate_through_real_parser_diamond(self): + # Independent behaviors, not sibling format implementations: one records + # physical acquisitions; the other scopes their names. Both cooperate + # through the real owning ABC and the existing ImageXpress leaf. + class RecordedAcquisition(GenericFilenameParser): + def __init__(self, *args, **kwargs): + self.acquisition_calls = [] + super().__init__(*args, **kwargs) + + def construct_acquisition_filename(self, components, **options): + name = super().construct_acquisition_filename(components, **options) + self.acquisition_calls.append(name) + return name + + class ScopedAcquisition(GenericFilenameParser): + def construct_acquisition_filename(self, components, **options): + return "scope172_" + super().construct_acquisition_filename(components, **options) + + class RecordedScopedImageXpressParser( + RecordedAcquisition, ScopedAcquisition, ImageXpressFilenameParser + ): + pass + + class ScopedRecordedImageXpressParser( + ScopedAcquisition, RecordedAcquisition, ImageXpressFilenameParser + ): + pass + + for parser_type in (RecordedScopedImageXpressParser, ScopedRecordedImageXpressParser): + with self.subTest(parser_type=parser_type.__name__): + parser = parser_type() + self.assertIs(FilenameParser.__registry__[parser_type.__name__], parser_type) + self.assertEqual(parser_type.__mro__.count(GenericFilenameParser), 1) + self.assertEqual(parser.FILENAME_COMPONENTS, tuple(AllComponents)) + components = parser.bind_declared_values( + ((AllComponents.WELL, "D12"), (AllComponents.SITE, 17), + (AllComponents.CHANNEL, 7), (AllComponents.Z_INDEX, 99), + (AllComponents.TIMEPOINT, 3))) + physical_name = parser.construct_acquisition_filename(components) + self.assertEqual(physical_name, "scope172_D12_s017_w7.tif") + self.assertEqual(len(parser.acquisition_calls), 1) + expected_record = (physical_name if parser_type is RecordedScopedImageXpressParser + else "D12_s017_w7.tif") + self.assertEqual(parser.acquisition_calls, [expected_record]) + # The ancestor's canonical operation still carries every axis. + canonical = GenericFilenameParser.construct_acquisition_filename(parser, components) + self.assertEqual(canonical, "D12_s017_w7_z099_t003.tif") + plate = self.root / parser_type.__name__ + plate.mkdir() + tifffile.imwrite(plate / physical_name, np.full((32, 32), 172, dtype=np.uint16)) + projection = SourcePlaneProjection( + OpenHCSPlaneAddress.from_component_values(components.declared_values()), + SourcePixelRef("disk", physical_name), + ) + # Real unchanged consumer and automatic declaration registry: + # no alternate formatter, registration edits or reader adapter. + metadata = SourceProjectionMetadataSerializer(parser).metadata_dict( + SourceProjectionSet((projection,)), microscope_handler_name="imagexpress", + source_filename_parser_name=parser_type.__name__, + grid_dimensions=[1, 1], pixel_size=0.65, main=True) + AtomicMetadataWriter().replace_subdirectory_metadata(get_metadata_path(plate), ".", metadata) + values = self.assert_coherent(plate, metadata, 1) + self.assertEqual(values[AllComponents.Z_INDEX], {"99"}) + self.assertEqual(values[AllComponents.TIMEPOINT], {"3"}) + + class RecordedScopedImageXpressHandler(ImageXpressHandler): + _microscope_type = "recorded_scoped_imagexpress" + + def __init__(self, filemanager, pattern_format=None): + super().__init__(filemanager, pattern_format) + self.parser = RecordedScopedImageXpressParser(filemanager, pattern_format) + + self.assertIs(MICROSCOPE_HANDLERS[RecordedScopedImageXpressHandler._microscope_type], + RecordedScopedImageXpressHandler) + plate = self.root / "declared-producer-extension" + with redirect_stdout(StringIO()): + generator = SyntheticMicroscopyGenerator( + str(plate), format="RecordedScopedImageXpress", grid_size=(1, 1), + tile_size=(32, 32), wavelengths=1, num_cells=4, wells=["D12"], + overlap_percent=0, stage_error_px=0, random_seed=7) + generator.generate_htd_file() + # Plane emission and metadata are the assigned identity surface; + # this does not certify new-family HTD/grid-layout generation. + generator._write_plane(np.full((32, 32), 172, dtype=np.uint16), + generator.timepoint_dir, "D12", 17, 7, 99) + generator.generate_openhcs_metadata(sub_dir=generator.timepoint_dir.name) + self.assertIsInstance(generator.parser, RecordedScopedImageXpressParser) + self.assertEqual(generator.parser.acquisition_calls, ["scope172_D12_s017_w7.tif"]) + self.assert_coherent(plate, self.document(plate), 1) + + +if __name__ == "__main__": + unittest.main(verbosity=2)