diff --git a/docs/validation/mixed-carrier-intensity-domain-20261004.rst b/docs/validation/mixed-carrier-intensity-domain-20261004.rst new file mode 100644 index 0000000000..ec41c3146f --- /dev/null +++ b/docs/validation/mixed-carrier-intensity-domain-20261004.rst @@ -0,0 +1,362 @@ +Mixed-carrier current-intensity ownership checkpoint +================================================== + +Active fix owner: Singer. Base main9f75ed8fbce41e6959b48c4961bc9cb612e6fd3b; +Root394 is merged, not an outstanding implementation promise. Current open +PR597 changes compilation/editor lifecycles and is disjoint from this family. +The existing Dewey/Root route was checked for an active conflicting claim. +This checkpoint publishes ownership and the determining relation; production +implementation, source controls and installed receiving are not yet qualified. + +Source-derived counterexample +---------------------------- + +NamedSourceBinding.apply_loaded_payload normalizes a declared RGB carrier +before converting it to monochrome. Scalar grayscale preserves integer codes. +ImagePayloadStackComposition stacks conformable members via np.stack, promoting +mixed integer/float storage without reconciling their current intensity units. +Per-plane metadata still carries acquisition scales and source dtypes. +normalize_image_payload_intensity scales integer arrays but simply casts float +arrays; source-plane selection does not restore the previous storage dtype. +Consequently a legal composition can change a scalar member's downstream +normalization result, even though its values and source metadata were retained. + +This is a source counterexample, not a replay of any original scientific job. +The retained original observations are pinned in the existing engineering +receipt BBBC007-CARRIER-INTENSITY-HANDOFF-20261004.rst under +engineering-neurite-units-20261003. They are not proof of the original run's +first wrong numeric transformation or its biological outcome. + +Required relation and owner closure +----------------------------------- + +Existing ImagePayloadMetadata must own the current analytical intensity domain +independently from acquisition dtype/full-scale provenance. The original +normalization recipe consumes that owned declaration through source loading, +composition and source-plane projection, then CellProfiler input/measurement +consumption and persisted metadata. Quantization proof is a separate fact: +losing exact quantization does not mean that normalized pixels became raw codes. +No consumer may infer current units solely from storage dtype or observed range. + +Read the complete declaration, constructor/write, composition, projection, +transformation, serialization and consumption family before implementation. +Use existing nominal owners and cooperative capabilities where independent +behavior composes; delete the replaced competing normalization decisions. +No caller-specific rescaling, guessed factor/codebook, min/max normalization, +parallel metadata store, codec or compatibility alias is admitted. + +Acceptance after coherent implementation +---------------------------------------- + +Tiny conformable declared scalar/RGB inputs must retain the same normalized +numerical units whether consumed independently, composed or reordered. +Include replicated and distinct RGB, explicit non-default source scale, +integer-only sources, raw floating analytical pixels, already normalized and +processed floating images, masks, source identity and persisted readback. +Require no double normalization and no loss of legitimate analytical remapping. +An independent declaration/capability must need only its own hook, not generic +consumer edits. Local source controls and ordinary installed/public receiving +are distinct readiness tiers; no original UNKNOWN input is replayed. + +NRA/refactor-audit and applicable catalog entries were read: IDEN-1 for current +versus acquisition units, BOUND-2 for bypassing the existing metadata owner, +IMPL-2/3 for external consumer dispatch and IMPL-12 for copied procedures. +Whole production/dependency AST and R1 closure are next, using existing tools. +The independent #541 physical/pixel analysis-frame route remains untouched. +All foreign gitlinks and retained untracked histories are preserved. No current +scientific package, input, settings, job or viewer has been changed. + +Authenticated source and implementation checkpoint +------------------------------------------------ + +Singer remains the active #599/#600 fix owner. On explicit public-source-fetch +authorization, the four exact recorded dependency commits were fetched from +their .gitmodules origins into the existing engineering599/source-references +directory, without changing shared dependency stores or creating a worktree: + +* ObjectState9fb5b7eeea96f3b7bb0579de30718bbcc0018823 +* PolyStore89deeef3662eabb11bc520fad9acd976698636bd +* pycodify108a8edbf25de258168756bac22c7572f018fa90 +* ZMQRuntime04d813fe6c93f74166c05847afb1eae158d3c817 + +The original audit Package parser dependency03 pass completed with 59, 114, +12 and 67 parsed modules respectively and zero parse/source failures. Original +source01 and dependency02 failures remain unchanged. The earlier donor claim +was incorrect: a grouped shell command concealed individual cat-file failures. +The exact source objects are now independently present; this is not a donor +wait or a claim that source01 originally had complete coverage. + +The existing ImageUnitIntervalIntensityMetadata record distinguishes normalized +analytical pixels from raw acquisition codes. An absent exact quantization +scale does not reset normalized pixels to raw. No second domain flag or store +is added. ImagePayloadMetadata now owns the numerical recipe; the original +normalization entrypoint delegates to it. The shared ImagePayloadStackComposition +ancestor reconciles mixed-domain members before dtype promotion, and its bundle +leaf delegates ordinary stacking through cooperative super(). The context-only +stacking procedure and all four production consumers are replaced together. +Native serialization that changes pixel values resets to the native raw domain; +value-preserving serialization retains the current analytical domain. + +This is the first production checkpoint, not a completed qualification. Tiny +conformable numerical, projection, native persistence, whole-context R1 and +installed/public receiving evidence are still required. Logs reside under +/home/ts/wt/openhcs-issue-batch-20260929/engineering599. + +Working source qualification and required receiving +-------------------------------------------------- + +Production05bcc19ecce31d7c1944f0f8cfe7e819534df33b is qualified below. Normal +integration of maina35d58163018785631f43ad7968e68b9569dc367 produced d6070360c: +its three incoming files are the published PolyStore floor, original spatial +field docstrings and their configuration-help control. All six #599 production +files are byte-identical to qualified05bcc. No foreign gitlink was changed. + +The original source9f75 numerical diagnostic completed in 1.02s/147908KiB, +terminal0 and Swap0. It demonstrates independent scalar128/255 = 0.5019608, +but mixed-stack selection = 128.0. Its original metadata module SHA is +278383795c5db8631f109a75833aad509f0f2070ac56fd585787793ed7b64220. +It is a real original-source numerical counterexample, not scientific input, +a mock product or replay of a retained scientific mutation. + +R0 initially found actual metadata-class growth +62 and a new foreign absence +probe +1. The correction acts on that owner, not the detector: the original five +intensity fields, quantization/projected-scale methods and numerical algorithm +now belong to ImagePayloadIntensityFields, an ABC composed into the existing +ImagePayloadMetadata. Existing SourceImageProvenanceFields owns replacement; +the declared MRO admits that concrete method ahead of the intensity contract. +The metadata leaf supplies the existing payload/leading-plane projection and +one axis-presence hook. Provenance, spatial placement and voxel spacing remain +their independent existing capabilities; no fields, stores or registries are +mirrored. The replaced methods and context-only composition procedure are gone. + +The source-binding monochrome hook retains declared source scale; normalized +state, not float dtype, prevents repeat scaling. The saved-buffer context owner +retargets acquisition facts onto the actual independent buffer's intensity +state, without replacing its pixels. Changing native file pixels returns the +metadata to that native raw domain; value-preserving storage retains current +units. A common exact quantization scale now requires every represented plane +to prove it, not merely the subset with known proofs. + +Source controls are terminal and original raw logs are retained: + +* Initial numeric controls01: 11 PASS/9 FAIL. Eight incorrect fixture assertions + required a composed diagnostic value_name to equal an original source label; + spatial origin, shape and fill remain the actual checked contract. One real + slotted-dataclass zero-argument super() failure was fixed through the original + cooperative super(ImagePayloadBundleContext, self) hook. +* Numeric/projection02: 98 PASS, 5.82s/310312KiB/Swap0. +* Consumer03: 74 PASS/3 FAIL, 156 deselected. These were remaining assertions of + the replaced normalized-domain marker after native pixel conversion; the + assertions now require raw-domain absence, not a weakened quantization test. +* Complete affected family04: 265 PASS, 66 deselected, 13.51s/439904KiB/Swap0. +* Family05 at fe3cc: 267 PASS, 67 deselected, 14.63s/438180KiB/Swap0. Real source + bindings, replicated/distinct RGB, reorderings, declared non-default scales, + raw integer/float carriers, analytical remapping, masks/calibration/provenance, + buffer identity and original image formats/readers are covered. +* Final changed quantization family06 at05bcc: 85 PASS, 5.31s/307920KiB/Swap0. + No unchanged broad suite was repeated for the narrow final proof correction. + +Two independent PixelAudit/MaskAudit capabilities cooperate in both C3 orders, +and a metadata normalization hook runs through normal generic composition. +The observed hook sequence is normalization, pixels, mask, with correct values. +The new cases add only their declarations/hooks, not generic consumer edits. + +Original R0 Git3b03785f45df2ef5dc62ba6aed99294192ecbb01, unchanged existing +run_pinned_r0_419.py and all six changed production paths: final04 PASS, +26.29s/87052KiB/Swap0, positive deltas empty; ImagePayloadMetadata god-class +excess decreases by77. The first Python3.12 invocation's unrelated original +package forward-annotation NameError is retained; existing Python3.14.7 executes +the same original detector/caller successfully, without source adaptation. + +Full audit Package AST at fe3cc parses 700 OpenHCS, 88 scripts, 156 benchmark, +697 test and 608 recorded dependency Python files with zero omissions. It emits +67/1/6/130 relevant full AST modules plus the original NumPy/skimage API ASTs. +Final05bcc's changed owner is parsed again through the original Package parser; +the remaining authenticated context and exact Git delta are unchanged. This +is source-family evidence, not behavioral or global-detector certification. + +Original full-context R1/NRA0844525 remains incomplete: its unchanged55s deadline +expired during parse_python_module, 58.25s total/139444KiB/Swap0. Prior reference +clone and incorrect NRA API bootstrap failures remain in distinct original +logs. Exact dependency source availability is resolved; no donor, detector copy, +raised deadline or omitted dependency is used to obtain a false global PASS. + +Next receiving boundary: one ordinary whole candidate package must exercise +declared tiny conformable scalar/RGB carriers through public source selection, +processing and persisted matched raw/result sampling with the same declared +scales, including swapped source order. Independent versus composed normalized +pixels must agree; analytical remapping must not be divided twice. This is +engineering-only, never the original scientific source/settings/job. Singer +owns receiving until normal package/lane custody is coordinated with the parent +and builder. No installed/native/MCP or biological acceptance is claimed here. + +The byte-exact source02 archive includes changed production/fixture sources, +original parser/bootstrap callers, original numerical failure, all failed and +passing controls/R0/R1 logs, and full source-family evidence. Original source01 +archive and all frozen scientific journals/UNKNOWN dispositions remain intact. + +Archive: docs/validation/mixed-carrier-intensity-source02-20261004.tar.gz, +SHA256 f060dbbbec0ace28daeedf24dee83d00cb3af717b10887d973b39daa24411031. + +Declared-memory and arithmetic-domain review correction +------------------------------------------------------ + +Parent review identified a reachable non-NumPy boundary: mixed intensity +reconciliation precedes the original runtime stack's host conversion. The +shared composition now resolves the original carrier's destination/device +before reconciliation, through one ancestor recipe also used by both pixel +hooks. The intensity owner uses the original ArrayBridge MemoryType.to_numpy +leaf before numerical operations; stack_runtime_slices still owns the final +conversion. Mask stacking derives its device from the output memory owner, +replacing the previous literal zero. No backend switch, converter or device +store is introduced. Applicable ownership findings are BOUND-2 and IMPL-12. + +The original complete-family AST remains archived. Supplemental original audit +Package AST in memory-owner-ast-before07.jsonl covers OpenHCS memory and the +entire CellProfiler interop family plus exact recorded ArrayBridge source. +CellProfilerCompileTime contracts install the existing runtime adapter; +CellProfilerModuleExecutor._image_request and ImageOutputRecorder.runtime_input_value +normalize declared image inputs, including named/special inputs. Measurement +inputs also use normalize_cellprofiler_image_payload. Threshold explicitly +normalizes at its numerical entrypoint. The arithmetic proof-clearing callers +are smoothing, illumination, morphology, thresholding, color, image math, +edge enhancement, image geometry, intensity and feature enhancement. Direct +helper calls do not establish the adapter invariant. Consequently the shared +without_unit_interval_intensity_scale method now invalidates only an existing +normalized proof: raw-domain arithmetic remains raw, not implicitly normalized. +No caller-specific normalization patches are added. + +Focused post-change memory/domain controls are next. They run the real original +MemoryType conversion and stack owners with a CPU-controlled CuPy leaf that +forbids implicit __array__, in both source orders, stack/bundle compositions, +implicit and explicit destinations, and nonzero device/mask preservation. +This source fixture cannot claim physical GPU or installed/public acceptance. +The previous 267/85 controls and original failure archives remain unchanged. + +Memory/domain controls07 retained 116 PASS/6 FAIL, 162 deselected, in +8.39s/339924KiB/Swap0. The six bundle failures were an incorrect fixture +declaration: it supplied STACK metadata while asserting BUNDLE's shared-mask +contract. Pixel values, destination/device and conversion guards all passed. +The fixture now declares the actual stack/bundle axis with each original +composition owner; mask values and device assertions remain intact. Only this +corrected affected slice is rerun, not the 116 unchanged passing controls. + +Controls08 retained 10 PASS/6 FAIL, 205 deselected, 6.48s/342020KiB/Swap0. +The first fixture diagnosis was incomplete: BUNDLE shared masks also require +the original SourceSpatialDomain's declared spatial extent. Without it, the +existing mask owner correctly returns one mask per member, even with BUNDLE +axis metadata. The corrected fixture covers both declared and absent extents, +requiring exact shared versus stacked masks accordingly. No production mask +rule or assertion is relaxed, and both original failed logs are preserved. + +Memory/domain closure at c8d2e65df +------------------------------- + +Production941f3c8e3 is unchanged by the subsequent fixture-only commits through +c8d2e65df. Controls09: 28 PASS, 205 deselected, 6.48s/341972KiB/Swap0 on one +CPU. These exercise the real ArrayBridge detector, MemoryType.to_numpy, +convert_memory, device scope and stack_runtime_slices, with only the external +CuPy module leaf controlled. Implicit __array__ is forbidden by the array +fixture. The destination, nonzero device, masks, both source orders, declared +and absent spatial extents, stack and bundle semantics all pass. Original raw +pixels and the external device selection remain unchanged. This is not a +physical GPU execution claim. + +Controls10: 2 PASS, 24 deselected, 4.86s/314148KiB/Swap0 on one CPU. The original +registered ImageOutputRecorder.runtime_input_value receives raw and already +normalized typed artifacts, returns the same normalized values and masks, +preserves quantization proof, and enters the correct arithmetic-domain state. +Together with the retained 116 passing controls07, this qualifies the narrow +memory/domain correction without rerunning the unchanged 267/85 suites. + +Original R0 memory05, unchanged detector/caller and complete changed path set +(six owned production files plus the integrated spatial-doc file), is terminal +PASS at c8d2e65df: 28.06s/88300KiB/Swap0, no positive deltas; metadata god-class +excess remains -77. The new shared composition memory recipe is below the +original excess threshold. No bound or detector was altered. + +The supplemental before AST parses 2 memory, 50 CellProfiler interop and 17 +ArrayBridge production modules, zero omissions. After11 parses 176 core and +640 unit modules and records complete AST/hash for both changed owners and +three changed fixtures, zero omissions, 9.97s/580444KiB/Swap0 on one CPU. +Its measured RSS is reported as measured, not rounded down to a 512MiB claim. +The earlier complete source/dependency closure and unchanged global R1 deadline +failure remain separate evidence; this supplemental parse is not global R1. + +The ordinary whole-package/public source-selection, processing and persisted +receiving boundary remains open and owned by Singer with the original parent +and builder. Existing fresh scientific families do not wait for this feature; +their immutable targets are not changed. No installation, native/client launch +or scientific mutation was performed in this correction. #541 pixel-unit +analysis remains an independent active owner, not a superseded Root394 hold. + +Byte-exact memory03 archive: +docs/validation/mixed-carrier-intensity-memory03-20261004.tar.gz, +SHA256 dd2a41387ae495b1677d8ba1d3ca9d9a5dda185ab1fb4cf4093c71c98b5218c3. +It retains both failed fixture receipts, final passing controls, unchanged +original R0 evidence, source/bootstrap callers and the before/after ASTs. + +Ordinary whole package04 receiving checkpoint +--------------------------------------------- + +Pinned source c63af64a604df4d940bfa716c6edce565ec82d47 was archived through +the original receiving06/prepare-source05 owner and built with the existing +setuptools builder (offline/no-isolation). The existing pip target materializer +installed the complete wheel and the previously qualified ObjectState1.1.9, +python-introspect0.1.16 and PolyStore0.3.2 wheels into a new unused private target: +/home/ts/wt/openhcs-issue-batch-20260929/engineering599/package01/target. +No new environment, dependency download, foreign checkout or SCI mutation. +Build/install terminal0, 8.79s/148604KiB/Swap0 on one CPU. Wheel SHA256 +772705f72b6adda29d26016cf322680527a727331e68f171b2ed89ad5488f7a7. + +The original whole receiving04 verifier is PASS:811 tracked source payloads, +789 Python,910 wheel RECORD entries, complete installed RECORD of all four +distributions, every declaration-selected knowledge projection and all13 +canonical skill files. Manifest-selected untracked benchmark assets are copied +only after exact equality with the original qualified06 wheel; SOURCE-ORIGIN.json +records each authentic origin. Native extensions are built from the original +declared source by setuptools; their whole wheel/installed payloads match RECORD. + +Installed affected-family control is terminal0:26 PASS in5.07s. Imports are +asserted under this complete target before tests; source checkout tests do not +replace installed modules. This requalifies the actual CPU/MRO/projection/storage +and CellProfiler normalization family; physical GPU execution is not claimed. + +Tiny public packet public-packet01 declares one12x15 scalar TIFF and one12x15x3 +RGB TIFF through NamesAndTypes plus the ordinary SourceBindingsConfig owner. +Four registered ImageMath outputs compare independent scalar/monochrome values +against mixed composition in both operand orders. Public inventory and full +persisted samples must demonstrate the expected normalized arithmetic, source +identity and .5 micrometer calibration. No viewer, science or private decoder. +The next original recorded-client/native lease is coordinated directly with +Dewey; this checkpoint is whole-installed proof, not public execution acceptance. + +Byte-exact builder/control/packet archive: +docs/validation/mixed-carrier-intensity-package04-20261004.tar.gz. +All prior source failures and uncertainty dispositions remain unchanged. + +Final ordinary installed/public acceptance06 +------------------------------------------- + +The historical pending receiving status above is now superseded. One whole +combined source422/target02 incorporates this production plus614; whole RECORD +proof and installed controls PASS. One original public client/native lease +completed the named-artifact-only CP workflow, inventory and ALL720 intensity +values exactly, including both arithmetic operand orders and source calibration. +The same lease completed SourceBindings→positions→assembly614, ALL560 mosaic +values and exact nine saved source contributors per channel. Original canonical +native VALUES observations validate their declared artifact/axis expectations. +See mixed-carrier-position-public-receiving06-20261004.rst and byte-exact sibling +archive SHA25612638e8d8b7b5dcd126f0999d070eb45a395999c5b43d7923c5b3ed440c5f264. + +Automatic final Reverse image publication still failed in the earlier original +job. Historical435 is closed for its accepted aggregate/Mosaic cases; this new +variant is not silently assigned to closed435 or proven600-independent. Its +exact trace and unchanged writer-family source evidence are preserved for a +named follow-up, without holding the accepted named-artifact-only workflow. +Original native close ACK/exited/succeeded; native and MCP gone, all ports empty. +Original CLI terminal2 is retained due to earlier negative commands/job, not0. + +Normal main09edf6cac integration changes none of this PR's six production +files compared with received422. Newer main/dependency changes are not reported +as installed/public-tested here. No repeated suite, rebuild or SCI hot update. diff --git a/docs/validation/mixed-carrier-intensity-memory03-20261004.tar.gz b/docs/validation/mixed-carrier-intensity-memory03-20261004.tar.gz new file mode 100644 index 0000000000..1fa19f81bb Binary files /dev/null and b/docs/validation/mixed-carrier-intensity-memory03-20261004.tar.gz differ diff --git a/docs/validation/mixed-carrier-intensity-package04-20261004.tar.gz b/docs/validation/mixed-carrier-intensity-package04-20261004.tar.gz new file mode 100644 index 0000000000..323bf6b6e0 Binary files /dev/null and b/docs/validation/mixed-carrier-intensity-package04-20261004.tar.gz differ diff --git a/docs/validation/mixed-carrier-intensity-source01-20261004.tar.gz b/docs/validation/mixed-carrier-intensity-source01-20261004.tar.gz new file mode 100644 index 0000000000..a3abb31544 Binary files /dev/null and b/docs/validation/mixed-carrier-intensity-source01-20261004.tar.gz differ diff --git a/docs/validation/mixed-carrier-intensity-source02-20261004.tar.gz b/docs/validation/mixed-carrier-intensity-source02-20261004.tar.gz new file mode 100644 index 0000000000..b4c6d6f357 Binary files /dev/null and b/docs/validation/mixed-carrier-intensity-source02-20261004.tar.gz differ diff --git a/docs/validation/mixed-carrier-position-public-receiving06-20261004.rst b/docs/validation/mixed-carrier-position-public-receiving06-20261004.rst new file mode 100644 index 0000000000..1a4533b31e --- /dev/null +++ b/docs/validation/mixed-carrier-position-public-receiving06-20261004.rst @@ -0,0 +1,140 @@ +Combined600/614 ordinary installed and public receiving PASS +========================================================== + +One complete ordinary wheel, source42203213d744b2b0393c9966868e8d02ec45c3d9; +wheelSHA ebfb65fe1b70741b322d4463d45f2cbd421381c9e603ba3c8240f05a64c1843c. +Original package02/target is immutable. Whole-byte/RECORD/source/resource proof +and installed3position controls retain their earlier published qualification. +The26 installed600 controls remain qualified at identical600 production bytes. + +One original public CLI PTY76599, MCP446810, invocation +acbd7b2fa1de48dc83c52a4aeca5e552, firstepoch1791115322. Health resolved target02. +One original public native bootstrap447841/create1791115349.67 on6012TCP/7012ACK. +Original catalog preparation READY sequence988. No viewer, science contact, +scientific source, private algorithm, installed edits, second client or timeout +change. The original recorded owner's fresh admission PASS preceded launch. + +600 actual path +--------------- + +Original two TIFFs were copied byte-exact into the declared writable workspace +before normal SourceBindings preparation. Public artifact-plan resolved scalar +Gray/channel1 and RGB Color/channel2 with physical .5/.5micrometer calibration. +The original importer merged two independent ImageMath declarations into one +step; three native steps produced four named image artifacts. + +Original session1/job1 compilation COMPLETE. Its job3 execution +07355a20-3504-4c67-aad0-dcb8f868db70 FAILED on automatic final image publication: +conflicting metadata for persisted Reverse occurrence. Full original traceback, +negative job and partial outputs remain. This is not a600 normalization success +receipt and is not claimed fixed by either600 or614; it reaches the shared +publication family, not a numerical exception or timeout. Historical435 is now +closed for its accepted cases, so this distinct Reverse variant is NOT described +as an unresolved435 fix or automatically proven independent of600. + +Distinct complete pipeline-artifacts-only02.py keeps identical acquisitions, +importer, four ImageMath declarations and named-artifact persistence. It changes +only the supported PathPlanningConfig well_filter=0 (automatic final main-flow +persistence disabled) and distinct output destination. No metadata guard is +relaxed. Public plan confirms named persistence enabled and automatic main-flow +persistence disabled. Session3/job5 compilation COMPLETE; job6 execution +a80fe53c-3270-4ca2-8b3c-d02da65eda04 COMPLETE. + +Public inventory exposed all four exact saved images. Native-resolution public +samples return180float32 values each: scalar/255, original skimage rgb2gray of +RGB/255, their average in each operand order. All720 values have ZERO absolute +error against independently computed expected arrays. Both arithmetic orders +agree. Public source calibration .5/.5micrometers remains. The existing named +image contract retains first-operand source identity (Gray for Forward, Color +for Reverse); no new dual-operand provenance capability is claimed. + +614 actual path +--------------- + +Original18embedded-header acquisition TIFFs (nine shuffled sites, two channels) +were copied byte-exact into the same declared workspace. Public SourceBindings +plan exposes all18 exact source planes, signed coordinates(-2,-1) through(12,9), +and .25Y/.5Xmicrometer calibration. Original registered acquisition_tile_positions +and assemble_stack_cpu use the native compiled positions artifact edge. +Session2/job2 compilation COMPLETE; job4 execution +b02b921e-5830-4ec5-aa7d-eb22288e1b22 COMPLETE. + +Canonical native RuntimeExecutionObservation export validates its own execution +and artifact expectations. Its two scoped positions records contain all nine +expected XY coordinates in resolvedsite order, distinct native channel1/2. +Native log records both registered calls and the original positions artifact +input join for assembly. Saved JSON positions agree; saved images and checkpoints +are exposed in the ordinary public inventory. + +Both public full14x20uint16 mosaics match ALL560 elements exactly: every4x6 tile +has value10*channel+site at(row*5,col*7), remaining holes zero. Public calibration +.25/.5micrometers preserved. Original VirtualWorkspaceSourceProjection decodes +saved image metadata: exactly nine source contributors/channel, original site +order1..9, exact signed positions and paths to the prepared source acquisition. +SpecialArtifact positions stores a list plus scoped identity, not ImageMetadata. +The image projection is the authoritative owner of saved pixel provenance. + +Verification and preserved negatives +----------------------------------- + +verify-received02.py terminal0,3.40s/287932KiB/CPU99%/Swap0. It uses original +ZMQRuntimeExecutionObservationExport.read/require_valid_observation and original +VirtualWorkspaceSourceProjection, no alternate runtime codec. Both observations +PASS; public arrays, source paths and all20 original source-file hashes PASS. + +verify-received01.py terminal1 remains: its incorrect assumption that the special +positions list carries ImagePayloadMetadata was replaced by the real saved-image +projection checks, not removed provenance assertions. Its original logs/script +remain. CLI `context` usage error, undeclared get_job_status name and read-only +plate writable-policy rejection remain in the complete original journal; none +dispatched an uncertain native mutation. Correct current entrypoints were used. + +Closure +------- + +Exactly one public close of the original native handle returned acknowledged, +process_exited, endpoint_terminated and succeeded alltrue, errors[]. Native447841 +and MCP446810 subsequently absent;6012/7012/6013/7013 listeners empty. No viewer +was started. Original CLI exit once; actual PTY76599 terminal2, attributable to +the retained earlier negative commands/jobs, not hidden or rewritten as0. +No UNKNOWN replay; native numeric OS exit not inferred from ACK. + +This qualifies the synthetic source-selection/processing/persistence numerical +and acquisition-position workflow. It does not prove biological usefulness, +physicalGPU execution, globalNRA R1 completion or repair of automatic final +publication. Current main later carries disjoint cancelled-dispatch, operations +observation and FAST-speckle changes; receiving remains explicitly pinned422. +Future complete packages must incorporate normal main/dependency floors; this +receipt is not mislabelled as a live test of those unrelated newer components. + +Determining publication attribution +---------------------------------- + +Actual failure is original function_outputs.py:942, metadata-writer comparison +of automatic produced projection with the named artifact at the same pathname. +That whole file is byte-identical before600 at main634875dd and received422: +Gitblob82eb17a7e63d54dbe2727dfca236566afb0ec1aa. source_projection.py and +interop/cellprofiler/runtime/main_flow.py also have no delta across those pins. +Historical pre600 engineering494 native execution9292bfb8 on2026-10-03 failed +at the same comparison/message after artifact publication; exact original log +engineering494/POINTS494_95/author-workspace/output/runtime/scratch/data/openhcs/ +logs/openhcs_zmq_server_port_6014_1791009392666738273.log:270..292 and original +mcp.stdout:2101/2142. Main's persisted-point-receiver-main-20261003.rst historically +records Root435 publication as unresolved, distinct from its accepted reopen. +Current435 comments show the aggregate and Mosaic cases were subsequently +fixed/accepted in merged394. The old statement is not current owner status. +This establishes a pre-existing publication family and unchanged writer guard; +it does NOT prove that this exact Reverse fixture fails on a pre600 runtime. +No older runtime was launched/replayed and no600-specific interaction is waived. + +Durable original evidence +------------------------- + +Original complete journals, typed native VALUES, all tiny source TIFFs, pipeline +documents, public pixel verification scripts/negative/PASS logs and produced +metadata are archived byte-exact in the sibling +mixed-carrier-position-public06-20261004.tar.gz. No original log was rewritten. +Persistent original receiving root remains +/home/ts/wt/openhcs-issue-batch-20260929/engineering599/public94-combined02. +Original VALUES paths are ENGINEERING600614_94/author-workspace/output/ +intensity-values02.json and positions-values01.json under that root. diff --git a/docs/validation/mixed-carrier-position-public06-20261004.tar.gz b/docs/validation/mixed-carrier-position-public06-20261004.tar.gz new file mode 100644 index 0000000000..e6bb5de5d6 Binary files /dev/null and b/docs/validation/mixed-carrier-position-public06-20261004.tar.gz differ diff --git a/openhcs/core/aligned_image_payload.py b/openhcs/core/aligned_image_payload.py index ab5aa3cda5..02dfc654af 100644 --- a/openhcs/core/aligned_image_payload.py +++ b/openhcs/core/aligned_image_payload.py @@ -396,21 +396,6 @@ def resolve_source_spatial_value(self, value: Any) -> Any: return adapter.value_in_payload_domain(reference) -def stack_image_payload_context( - image_payloads: Sequence[Any], - stack: RuntimeArrayData, - *, - metadata_mode: ImagePayloadMetadataCompositionMode, -) -> Any: - """Attach composed image metadata and masks to a freshly stacked payload.""" - payloads = tuple(image_payloads) - metadata = ImagePayloadMetadata.compose( - payloads, - mode=metadata_mode, - ) - return metadata.payload_with(stack, _stack_image_payload_mask(payloads, stack)) - - class ImagePayloadStackComposition(ABC): """Compose pixels, masks and provenance on one declared leading axis.""" @@ -474,13 +459,10 @@ def from_loaded_images( workspace_source_lookups ) if metadata_mode is ImagePayloadMetadataCompositionMode.STACK: - main_data_stack = stack_runtime_slices( - tuple(image_payload_data(payload) for payload in payloads), - execution_plan.input_memory_type, - execution_plan.device_id_for(execution_plan.input_memory_type), - ) - main_data_stack = stack_image_payload_context( - payloads, main_data_stack, metadata_mode=metadata_mode, + main_data_stack = stack_image_payloads( + payloads, metadata_mode=metadata_mode, + memory_type=execution_plan.input_memory_type, + device_id=execution_plan.device_id_for(execution_plan.input_memory_type), ) elif metadata_mode is ImagePayloadMetadataCompositionMode.BUNDLE: main_data_stack = ImagePayloadBundleContext.from_payloads( @@ -530,12 +512,13 @@ def with_saved_output_context( ) -> RuntimeArrayData: """Attach saved member context without replacing the independent buffer.""" data = image_payload_data(stack_payload) + current_intensity = image_payload_metadata(stack_payload) if ( len(payloads) == 1 and single_output_plane_axis is metadata[0].plane_axis and np.shape(data) == np.shape(image_payload_data(payloads[0])) ): - return metadata[0].replace_fields().payload_with( + return metadata[0].with_current_intensity_from(current_intensity).payload_with( data, image_payload_mask(stack_payload), ) if np.shape(data)[:1] != (len(payloads),): @@ -551,16 +534,21 @@ def with_saved_output_context( else ImagePayloadMetadataCompositionMode.STACK ) output_metadata = ImagePayloadMetadata.compose( - tuple(payloads), mode=mode, source_metadata=tuple(metadata), + tuple(payloads), mode=mode, source_metadata=tuple( + record.with_current_intensity_from(current_intensity, plane_index=index) + for index, record in enumerate(metadata) + ), ) return output_metadata.payload_with( data, _stack_image_payload_mask(tuple(payloads), data), ) - def composition_source_metadata(self) -> tuple[ImagePayloadMetadata, ...]: + def composition_source_metadata( + self, payloads: Sequence[Any] | None = None, + ) -> tuple[ImagePayloadMetadata, ...]: return tuple( self.composition_payload_metadata(image_payload_metadata(payload)) - for payload in self.composition_payloads + for payload in (self.composition_payloads if payloads is None else payloads) ) def composition_payload_metadata( @@ -569,24 +557,47 @@ def composition_payload_metadata( """Preserve each input's declared provenance unless the axis owner projects it.""" return metadata - def compose(self) -> Any: - payloads = self.composition_payloads + @staticmethod + def composition_memory_domain( + payloads: tuple[RuntimeArrayData, ...], *, + memory_type: str | None = None, device_id: int | None = None, + ) -> tuple[str, int | None]: + """Resolve one composition destination through the original memory owner.""" + if memory_type is not None: + return memory_type, device_id + memory_type = detect_memory_type(payloads[0]) + return memory_type, MemoryType(memory_type).device_id_of(payloads[0]) + + def compose( + self, *, memory_type: str | None = None, device_id: int | None = None, + ) -> Any: + # Intensity reconciliation projects pixels to the numerical host domain. + # Resolve the destination from the original carrier before that projection. + memory_type, device_id = self.composition_memory_domain( + tuple(image_payload_data(payload) for payload in self.composition_payloads), + memory_type=memory_type, device_id=device_id, + ) + payloads = ImagePayloadMetadata.intensity_coherent_payloads(self.composition_payloads) composed = self.compose_unmasked( - tuple(image_payload_data(payload) for payload in payloads) + tuple(image_payload_data(payload) for payload in payloads), + memory_type=memory_type, device_id=device_id, ) metadata = ImagePayloadMetadata.compose( payloads, mode=self.composition_metadata_mode, - source_metadata=self.composition_source_metadata(), + source_metadata=self.composition_source_metadata(payloads), ) return metadata.payload_with(composed, self.compose_mask(composed, metadata)) def compose_unmasked( - self, payloads: tuple[RuntimeArrayData, ...] + self, payloads: tuple[RuntimeArrayData, ...], *, + memory_type: str | None = None, device_id: int | None = None, ) -> RuntimeArrayData: - memory_type = detect_memory_type(payloads[0]) + memory_type, device_id = self.composition_memory_domain( + payloads, memory_type=memory_type, device_id=device_id, + ) return stack_runtime_slices( - payloads, memory_type, MemoryType(memory_type).device_id_of(payloads[0]) + payloads, memory_type, device_id, ) def compose_mask(self, composed: Any, metadata: ImagePayloadMetadata) -> Any | None: @@ -622,10 +633,14 @@ def stack_image_payloads( image_payloads: Sequence[Any], *, metadata_mode: ImagePayloadMetadataCompositionMode, + memory_type: str | None = None, + device_id: int | None = None, ) -> Any: """Stack image payloads in their declared memory domain with full context.""" - return ImagePayloadStackContext(image_payloads, metadata_mode).compose() + return ImagePayloadStackContext(image_payloads, metadata_mode).compose( + memory_type=memory_type, device_id=device_id, + ) def _stack_image_payload_mask( @@ -671,7 +686,7 @@ def _stack_image_payload_mask( return stack_runtime_slices( resolved_masks, detect_memory_type(stack), - 0, + MemoryType(detect_memory_type(stack)).device_id_of(stack), ) @@ -854,10 +869,12 @@ def combined_mask(self) -> RuntimeArrayData | None: def compose_unmasked( self, payloads: tuple[RuntimeArrayData, ...], + *, memory_type: str | None = None, device_id: int | None = None, ) -> RuntimeArrayData: """Compose image payload arrays without mask/metadata wrapping.""" - memory_type = detect_memory_type(payloads[0]) - device_id = MemoryType(memory_type).device_id_of(payloads[0]) + memory_type, device_id = self.composition_memory_domain( + payloads, memory_type=memory_type, device_id=device_id, + ) channel_axes = tuple( metadata.normalized_source_channel_axis(payload) for payload, metadata in zip( @@ -868,7 +885,9 @@ def compose_unmasked( ) declared_channel_count = sum(axis is not None for axis in channel_axes) if declared_channel_count in {0, len(payloads)}: - return stack_runtime_slices(payloads, memory_type, device_id) + return super(ImagePayloadBundleContext, self).compose_unmasked( + payloads, memory_type=memory_type, device_id=device_id, + ) return self.compose_mixed_channel_payloads( payloads, channel_axes=channel_axes, @@ -1225,9 +1244,9 @@ def copy_projected_output_stack( return ImagePayloadBundleContext.from_payloads( payloads, metadata_mode=metadata_mode, ).compose() - stacked = stack_runtime_slices(data, memory_type, device_id) - return stack_image_payload_context( - payloads, stacked, metadata_mode=metadata_mode, + return stack_image_payloads( + payloads, metadata_mode=metadata_mode, + memory_type=memory_type, device_id=device_id, ) def output_values_for_artifact_specs( diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index 7fee966e4d..b5b1552315 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -797,8 +797,7 @@ def compose_runtime_values( return values[0].data from openhcs.core.aligned_image_payload import stack_image_payloads - from openhcs.core.aligned_image_payload import stack_image_payload_context - from openhcs.core.memory import detect_memory_type, stack_runtime_slices + from openhcs.core.memory import detect_memory_type from openhcs.core.runtime_image_values import ( ImagePayloadMetadataCompositionMode, image_payload_data, @@ -814,9 +813,8 @@ def compose_runtime_values( if producer_group_scope is not None: payloads = tuple(value.data for value in values) arrays = tuple(image_payload_data(payload) for payload in payloads) - return stack_image_payload_context( - payloads, - stack_runtime_slices(arrays, detect_memory_type(arrays[0]), 0), + return stack_image_payloads( + payloads, memory_type=detect_memory_type(arrays[0]), device_id=0, metadata_mode=ImagePayloadMetadataCompositionMode.for_plane_axis( RuntimePlaneAxis.RUNTIME_SLICE ), diff --git a/openhcs/core/image_file_serialization.py b/openhcs/core/image_file_serialization.py index d330a45be6..cb49cfa1c0 100644 --- a/openhcs/core/image_file_serialization.py +++ b/openhcs/core/image_file_serialization.py @@ -155,7 +155,8 @@ def project_image_metadata( raise ValueError("Saved image metadata requires an actual native dtype.") native_scale_governs = self.intensity_scale is not None or not values_preserved if not values_preserved: - metadata = metadata.without_unit_interval_intensity_scale().replace_fields( + metadata = metadata.replace_fields( + unit_interval_intensity=None, physical_border_edges_yx=None, mask_defines_border=None, ) diff --git a/openhcs/core/projected_image_output.py b/openhcs/core/projected_image_output.py index 0bbbbf3bce..2d9f65f41b 100644 --- a/openhcs/core/projected_image_output.py +++ b/openhcs/core/projected_image_output.py @@ -10,7 +10,7 @@ from openhcs.core.aligned_image_payload import ( AlignedImageStack, - stack_image_payload_context, + stack_image_payloads, ) from openhcs.core.registry_strategies import NominalTypeKeyedStrategyMixin from openhcs.core.runtime_array_values import ( @@ -236,9 +236,8 @@ def contextualize( output_slice, ) ) - return stack_image_payload_context( + return stack_image_payloads( tuple(contextualized_slices), - output_data, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, ) diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index a351979f8e..98b295d453 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -13,7 +13,7 @@ from typing import Any, TypeVar import numpy as np -from arraybridge import ArrayGeometry +from arraybridge import ArrayGeometry, MemoryType, detect_memory_type from python_introspect import dataclass_from_mapping from zmqruntime.viewer_protocol import ( ViewerWireField, @@ -62,7 +62,12 @@ @dataclass(frozen=True, slots=True) class ImageUnitIntervalIntensityMetadata: - """Authored proof that image pixels retain exact unit-interval quantization.""" + """Normalized analytical pixels, with optional exact quantization proof. + + An absent scale means arithmetic changed the quantization, not that current + pixels reverted to acquisition codes. Absence of this record denotes the + unnormalized source domain. + """ scale: int | None = None source_plane_scales: tuple[int | None, ...] = () @@ -100,19 +105,188 @@ def without_source_planes(self) -> "ImageUnitIntervalIntensityMetadata": return type(self)(scale=self.scale) +@dataclass +class ImagePayloadIntensityFields(ABC): + """Current pixel intensity semantics, independent of provenance and placement. + + The composed metadata owner supplies payload construction, field replacement + and leading-plane projection; this capability owns scale/quantization and + their numerical interpretation. Dataclass fields remain the single stored + declarations consumed by the original metadata codecs. + """ + + intensity_scale: float | None = None + source_dtype: str | None = None + unit_interval_intensity: ImageUnitIntervalIntensityMetadata | None = None + source_plane_intensity_scales: tuple[float | None, ...] = () + source_plane_dtypes: tuple[str | None, ...] = () + + @abstractmethod + def replace_fields(self, **changes: Any) -> "ImagePayloadMetadata": ... + + @abstractmethod + def payload_with(self, data: Any, mask: Any | None = None) -> Any: ... + + @abstractmethod + def for_leading_source_plane(self, index: int) -> "ImagePayloadMetadata": ... + + @property + @abstractmethod + def has_leading_intensity_axis(self) -> bool: ... + + @property + def has_normalized_intensity(self) -> bool: + """Current pixels have left the raw acquisition-code domain.""" + return self.unit_interval_intensity is not None + + @property + def unit_interval_intensity_scale(self) -> int | None: + """Return the authored scalar unit-interval quantization proof.""" + if self.unit_interval_intensity is None: + return None + return self.unit_interval_intensity.scale + + @property + def source_plane_unit_interval_intensity_scales(self) -> tuple[int | None, ...]: + """Return authored per-plane unit-interval quantization proofs.""" + if self.unit_interval_intensity is None: + return () + return self.unit_interval_intensity.source_plane_scales + + def intensity_scale_for_source_plane(self, plane_index: int) -> float | None: + """Return the best available intensity scale for one source plane.""" + plane_scale = _tuple_value(self.source_plane_intensity_scales, plane_index) + return self.intensity_scale if plane_scale is None else plane_scale + + def unit_interval_intensity_scale_for_source_plane(self, plane_index: int) -> int | None: + """Return the scale proving current pixels are exact integer/scale values.""" + if self.unit_interval_intensity is None: + return None + return self.unit_interval_intensity.scale_for_source_plane(plane_index) + + def project_intensity_proof(self, plane_index: int | None) -> ImageUnitIntervalIntensityMetadata | None: + """Select a plane's proof, or retain only the proof for the whole image.""" + if self.unit_interval_intensity is None: + return None + if plane_index is None: + return self.unit_interval_intensity.without_source_planes() + return self.unit_interval_intensity.for_source_plane(plane_index) + + def common_unit_interval_intensity_scale(self) -> int | None: + """Return the common unit-interval quantization proof for this payload.""" + if self.source_plane_unit_interval_intensity_scales: + present = tuple( + scale + for plane_index in range(len(self.source_plane_unit_interval_intensity_scales)) + for scale in (self.unit_interval_intensity_scale_for_source_plane(plane_index),) + ) + if any(scale is None for scale in present): + return None + first = present[0] + if all(scale == first for scale in present): + return first + return None + return self.unit_interval_intensity_scale + + def with_unit_interval_intensity_scale(self, scale: int | None) -> "ImagePayloadMetadata": + """Return metadata with the current unit-interval pixel proof updated.""" + return self.replace_fields( + unit_interval_intensity=ImageUnitIntervalIntensityMetadata(scale=scale) + ) + + def without_unit_interval_intensity_scale(self) -> "ImagePayloadMetadata": + """Invalidate quantization without changing the current intensity domain.""" + return self.replace_fields( + unit_interval_intensity=( + ImageUnitIntervalIntensityMetadata() if self.has_normalized_intensity + else None + ) + ) + + def with_current_intensity_from( + self, source: "ImagePayloadIntensityFields", *, plane_index: int | None = None, + ) -> "ImagePayloadMetadata": + """Retarget acquisition context onto an independently owned pixel buffer.""" + return self.replace_fields( + unit_interval_intensity=( + source.unit_interval_intensity if plane_index is None + else source.project_intensity_proof(plane_index) + ) + ) + + def normalize_intensity_payload( + self, payload: Any, *, dtype: Any = None, channel_index: int = 0, + ) -> Any: + """Normalize the declared current domain, independently of storage dtype.""" + data = image_payload_data(payload) + array = np.asarray(MemoryType(detect_memory_type(data)).to_numpy(data)) + target_dtype = np.dtype(np.float32 if dtype is None else dtype) + if not ( + np.issubdtype(array.dtype, np.number) + or np.issubdtype(array.dtype, np.bool_) + ) or np.issubdtype(array.dtype, np.complexfloating): + return payload + if self.has_normalized_intensity: + return self.payload_with( + array.astype(target_dtype, copy=False), image_payload_mask(payload), + ) + if self.has_leading_intensity_axis and self.source_plane_intensity_scales: + if len(self.source_plane_intensity_scales) != len(array): + raise ValueError( + "Image intensity scales must match the declared leading plane axis." + ) + planes = tuple( + self.for_leading_source_plane(index).normalize_intensity_payload( + plane, dtype=target_dtype, + ) + for index, plane in enumerate(array) + ) + proof = _ImagePayloadMetadataComposer.composed_unit_interval_intensity( + tuple(image_payload_metadata(plane) for plane in planes) + ) + return self.replace_fields(unit_interval_intensity=proof).payload_with( + np.stack(tuple(image_payload_data(plane) for plane in planes)), + image_payload_mask(payload), + ) + scale = self.intensity_scale_for_source_plane(channel_index) + if scale is None: + # Bare arrays are admitted at this original numerical boundary. A + # promoted float uses its declared source scale, never a range guess. + scale = image_intensity_scale_for_dtype(array.dtype) + normalized = array.astype(target_dtype, copy=False) + proof_scale = None + if scale is not None: + if not np.isfinite(scale) or scale <= 0: + raise ValueError("Source intensity scale must be finite and positive.") + normalized = normalized / float(scale) + if np.issubdtype(array.dtype, np.integer) and float(scale).is_integer(): + proof_scale = int(scale) + return self.with_unit_interval_intensity_scale(proof_scale).payload_with( + normalized, image_payload_mask(payload), + ) + + @classmethod + def intensity_coherent_payloads(cls, payloads: Sequence[Any]) -> tuple[Any, ...]: + """Reconcile raw/normalized members before a dense buffer erases dtype.""" + metadata = tuple(image_payload_metadata(payload) for payload in payloads) + normalized = tuple(record.has_normalized_intensity for record in metadata) + if not any(normalized) or all(normalized): + return tuple(payloads) + return tuple( + record.normalize_intensity_payload(payload) + for record, payload in zip(metadata, payloads, strict=True) + ) + + @dataclass(slots=True) class ImagePayloadMetadata( SourceImageProvenanceFields, + ImagePayloadIntensityFields, SourceSpatialDomainFields, SourceVoxelSpacingFields, ): """Generic source-image metadata that should travel with runtime pixels.""" - intensity_scale: float | None = None - source_dtype: str | None = None - unit_interval_intensity: ImageUnitIntervalIntensityMetadata | None = None - source_plane_intensity_scales: tuple[float | None, ...] = () - source_plane_dtypes: tuple[str | None, ...] = () physical_border_edges_yx: PhysicalBorderEdgesYX = None mask_defines_border: bool | None = None source_channel_axis: int | None = field( @@ -302,24 +476,6 @@ def has_values(self) -> bool: ) ) - @property - def unit_interval_intensity_scale(self) -> int | None: - """Return the authored scalar unit-interval quantization proof.""" - - if self.unit_interval_intensity is None: - return None - return self.unit_interval_intensity.scale - - @property - def source_plane_unit_interval_intensity_scales( - self, - ) -> tuple[int | None, ...]: - """Return authored per-plane unit-interval quantization proofs.""" - - if self.unit_interval_intensity is None: - return () - return self.unit_interval_intensity.source_plane_scales - def attach_to(self, payload: Any) -> RuntimeArrayData: """Attach this metadata to an existing image payload.""" return self.payload_with( @@ -697,30 +853,6 @@ def source_image_paths(self) -> tuple[str, ...]: ) ) - def intensity_scale_for_source_plane(self, plane_index: int) -> float | None: - """Return the best available intensity scale for one source plane.""" - plane_scale = _tuple_value(self.source_plane_intensity_scales, plane_index) - return self.intensity_scale if plane_scale is None else plane_scale - - def unit_interval_intensity_scale_for_source_plane( - self, - plane_index: int, - ) -> int | None: - """Return the scale proving current pixels are exact integer/scale values.""" - if self.unit_interval_intensity is None: - return None - return self.unit_interval_intensity.scale_for_source_plane(plane_index) - - def project_intensity_proof( - self, plane_index: int | None - ) -> ImageUnitIntervalIntensityMetadata | None: - """Select a plane's proof, or retain only the proof for the whole image.""" - if self.unit_interval_intensity is None: - return None - if plane_index is None: - return self.unit_interval_intensity.without_source_planes() - return self.unit_interval_intensity.for_source_plane(plane_index) - @property def source_plane_metadata_count(self) -> int: """Return the cardinality represented by plane-specific metadata.""" @@ -744,27 +876,6 @@ def source_metadata_by_payload(self) -> tuple["ImagePayloadMetadata", ...]: for plane_index in range(self.source_plane_metadata_count) ) - def common_unit_interval_intensity_scale(self) -> int | None: - """Return the common unit-interval quantization proof for this payload.""" - if self.source_plane_unit_interval_intensity_scales: - present = tuple( - int(scale) - for plane_index in range( - len(self.source_plane_unit_interval_intensity_scales) - ) - for scale in ( - self.unit_interval_intensity_scale_for_source_plane(plane_index), - ) - if scale is not None - ) - if not present: - return None - first = present[0] - if all(scale == first for scale in present): - return first - return None - return self.unit_interval_intensity_scale - def payload_with(self, data: Any, mask: Any | None = None) -> Any: """Return image payload data carrying this metadata.""" if mask is None and self.has_values: @@ -941,20 +1052,10 @@ def for_grouped_source_plane_projection( return metadata return metadata.for_source_planes(source_plane_indices) - def with_unit_interval_intensity_scale( - self, - scale: int | None, - ) -> "ImagePayloadMetadata": - """Return metadata with the current unit-interval pixel proof updated.""" - return self.replace_fields( - unit_interval_intensity=ImageUnitIntervalIntensityMetadata(scale=scale) - ) - - def without_unit_interval_intensity_scale(self) -> "ImagePayloadMetadata": - """Return metadata after an arithmetic transform changed pixel values.""" - return self.replace_fields( - unit_interval_intensity=ImageUnitIntervalIntensityMetadata(), - ) + @property + def has_leading_intensity_axis(self) -> bool: + """Bind intensity projection to the existing image-axis declaration.""" + return self.plane_axis is not None def without_spatial_domain(self) -> "ImagePayloadMetadata": """Return metadata with invalidated source-spatial placement removed.""" @@ -1918,31 +2019,10 @@ def normalize_image_payload_intensity( dtype: Any = None, channel_index: int = 0, ) -> Any: - """Normalize image pixels by payload metadata while preserving context.""" - import numpy as np - - array = np.asarray(image_payload_data(payload)) - target_dtype = np.float32 if dtype is None else np.dtype(dtype) - if np.issubdtype(array.dtype, np.bool_): - normalized = array.astype(target_dtype) - elif np.issubdtype(array.dtype, np.integer): - intensity_scale = image_payload_intensity_scale( - payload, - channel_index=channel_index, - ) - if intensity_scale is None or intensity_scale <= 1: - normalized = array.astype(target_dtype) - else: - normalized = array.astype(target_dtype) / float(intensity_scale) - metadata = image_payload_metadata( - payload - ).with_unit_interval_intensity_scale(int(intensity_scale)) - return with_image_payload_data(payload, normalized, metadata=metadata) - elif np.issubdtype(array.dtype, np.floating): - normalized = array.astype(target_dtype, copy=False) - else: - return payload - return with_image_payload_data(payload, normalized) + """Enter the metadata-owned normalization recipe once at the array boundary.""" + return image_payload_metadata(payload).normalize_intensity_payload( + payload, dtype=dtype, channel_index=channel_index, + ) def _tuple_value(values: tuple[Any, ...], index: int) -> Any | None: diff --git a/openhcs/core/source_bindings.py b/openhcs/core/source_bindings.py index 1cf4eae158..96dd16d3ad 100644 --- a/openhcs/core/source_bindings.py +++ b/openhcs/core/source_bindings.py @@ -1070,7 +1070,6 @@ def apply_loaded_payload( source_channel_axis = None metadata = metadata.replace_fields( unit_interval_intensity=ImageUnitIntervalIntensityMetadata(), - intensity_scale=ImagePayloadMetadata.for_array(data).intensity_scale, ) if self.load_as_mask: data = np.asarray(data, dtype=bool) diff --git a/tests/unit/test_aligned_image_payload_device.py b/tests/unit/test_aligned_image_payload_device.py index db18f9fa59..69ec625c23 100644 --- a/tests/unit/test_aligned_image_payload_device.py +++ b/tests/unit/test_aligned_image_payload_device.py @@ -1,9 +1,140 @@ import numpy as np +import pytest +import sys +from types import SimpleNamespace from arraybridge import MemoryType import openhcs.core.aligned_image_payload as aligned_image_payload -from openhcs.core.aligned_image_payload import ImagePayloadBundleContext +from openhcs.core.aligned_image_payload import ( + ImagePayloadBundleContext, ImagePayloadStackContext, +) +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, ImagePayloadMetadataCompositionMode, + ImageUnitIntervalIntensityMetadata, image_payload_data, image_payload_mask, + image_payload_metadata, +) +from openhcs.core.source_spatial_domain import SourceSpatialDomain + + +@pytest.fixture +def declared_cupy_leaf(monkeypatch): + """CPU-controlled external leaf; real MemoryType/converters run unchanged.""" + state = {'device': 0, 'downloads': [], 'uploads': []} + + class DeviceScope: + def __init__(self, device_id): + self.device_id = device_id + + def __enter__(self): + self.previous = state['device'] + state['device'] = self.device_id + + def __exit__(self, *exc): + state['device'] = self.previous + + class DeviceArray: + __module__ = 'cupy' + + def __init__(self, values, device_id=1): + self.values = np.asarray(values) + self.device = SimpleNamespace(id=device_id) + + @property + def shape(self): + return self.values.shape + + @property + def dtype(self): + return self.values.dtype + + @property + def ndim(self): + return self.values.ndim + + def __array__(self, dtype=None, copy=None): + raise TypeError('Implicit device-to-host conversion is forbidden') + + def get(self): + state['downloads'].append(self.device.id) + return self.values + + def __getitem__(self, key): + return type(self)(self.values[key], self.device.id) + + def astype(self, dtype, copy=False): + return type(self)(self.values.astype(dtype, copy=copy), self.device.id) + + def upload(values): + state['uploads'].append(state['device']) + return DeviceArray(values, state['device']) + + monkeypatch.setitem(sys.modules, 'cupy', SimpleNamespace( + cuda=SimpleNamespace( + runtime=SimpleNamespace(getDeviceCount=lambda: 2), Device=DeviceScope, + ), + array=upload, + logical_and=lambda left, right: DeviceArray( + np.logical_and(left.values, right.values), state['device'], + ), + )) + return DeviceArray, state + + +@pytest.mark.parametrize('composition,mode', ( + (ImagePayloadStackContext, ImagePayloadMetadataCompositionMode.STACK), + (ImagePayloadBundleContext, ImagePayloadMetadataCompositionMode.BUNDLE), +)) +@pytest.mark.parametrize('raw_first', (False, True)) +@pytest.mark.parametrize('destination', ('implicit', 'numpy', 'cupy')) +@pytest.mark.parametrize('declared_spatial_extent', (False, True)) +def test_mixed_intensity_composition_uses_declared_memory_conversion( + declared_cupy_leaf, composition, mode, raw_first, destination, declared_spatial_extent, +): + DeviceArray, state = declared_cupy_leaf + spatial_domain = SourceSpatialDomain( + source_shape_yx=(2, 3) if declared_spatial_extent else None, + ) + raw = ImagePayloadMetadata( + intensity_scale=64, source_dtype='uint8', source_spatial_domain=spatial_domain, + ).payload_with( + DeviceArray(np.full((2, 3), 32, dtype=np.uint8)), + DeviceArray(np.array([[True, False, True], [False, True, True]])), + ) + normalized = ImagePayloadMetadata( + intensity_scale=64, source_dtype='uint8', + unit_interval_intensity=ImageUnitIntervalIntensityMetadata(scale=64), + source_spatial_domain=spatial_domain, + ).payload_with(DeviceArray(np.full((2, 3), 0.5, dtype=np.float32)), + DeviceArray(np.ones((2, 3), dtype=bool))) + inputs = (raw, normalized) if raw_first else (normalized, raw) + context = composition(inputs, metadata_mode=mode) + kwargs = {} if destination == 'implicit' else { + 'memory_type': destination, 'device_id': 1 if destination == 'cupy' else None, + } + result = context.compose(**kwargs) + output = image_payload_data(result) + requested = 'cupy' if destination == 'implicit' else destination + owner = MemoryType(requested) + np.testing.assert_array_equal(owner.to_numpy(output), 0.5) + assert owner.device_id_of(output) == (1 if requested == 'cupy' else None) + assert image_payload_metadata(result).has_normalized_intensity + mask = image_payload_mask(result) + mask_owner = MemoryType(aligned_image_payload.detect_memory_type(mask)) + masks = mask_owner.to_numpy(mask) + expected = raw.mask.values + if composition is ImagePayloadStackContext or not declared_spatial_extent: + expected = np.stack(tuple(value.mask.values for value in inputs)) + # Bundle's non-shared mask preserves its mask owner's device; dense stack + # masks are returned on the declared output device through the ancestor. + if composition is ImagePayloadStackContext: + assert mask_owner.device_id_of(mask) == owner.device_id_of(output) + np.testing.assert_array_equal(masks, expected) + assert state['downloads'] + assert all(device == 1 for device in state['downloads']) + assert all(device == 1 for device in state['uploads']) + assert state['device'] == 0 + np.testing.assert_array_equal(raw.data.values, 32) def test_image_bundle_stacks_on_the_payload_framework_device(monkeypatch) -> None: diff --git a/tests/unit/test_cellprofiler_full_stack_materialization.py b/tests/unit/test_cellprofiler_full_stack_materialization.py index d4fd62b96b..fa75d153b0 100644 --- a/tests/unit/test_cellprofiler_full_stack_materialization.py +++ b/tests/unit/test_cellprofiler_full_stack_materialization.py @@ -368,9 +368,9 @@ def test_independent_composition_capabilities_cooperate_in_both_mro_orders(rever calls = [] class PixelCapability: - def compose_unmasked(self, payloads): + def compose_unmasked(self, payloads, **kwargs): calls.append("pixels") - return super().compose_unmasked(payloads) + 1 + return super().compose_unmasked(payloads, **kwargs) + 1 class MaskCapability: def compose_mask(self, composed, metadata): diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index f858658d81..c00a5891c8 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -84,7 +84,6 @@ AlignedImageSliceContext, AlignedImageStack, pack_aligned_image_outputs, - stack_image_payload_context, stack_image_payloads, unstack_image_payload_context, ) @@ -798,9 +797,8 @@ def test_stack_payload_context_projects_single_payload_mask_to_stack_domain(): data, mask ) - stacked = stack_image_payload_context( + stacked = stack_image_payloads( (payload,), - stack, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, ) @@ -859,9 +857,8 @@ def test_stack_payload_context_preserves_single_volumetric_payload_identity(): ) ).payload_with(data, None) - stacked_payload = stack_image_payload_context( + stacked_payload = stack_image_payloads( (payload,), - data[np.newaxis, ...], metadata_mode=ImagePayloadMetadataCompositionMode.STACK, ) stacked_metadata = image_payload_metadata(stacked_payload) @@ -892,9 +889,8 @@ def test_stack_payload_context_nests_incompatible_singleton_plane_topology(): plane_axis=RuntimePlaneAxis.SOURCE_BINDING, ).payload_with(data, None) - stacked_payload = stack_image_payload_context( + stacked_payload = stack_image_payloads( (payload,), - data[np.newaxis, ...], metadata_mode=ImagePayloadMetadataCompositionMode.STACK, ) provenance_planes = image_payload_metadata( @@ -919,9 +915,8 @@ def test_stack_payload_context_preserves_singleton_stack_payload_mask_domain(): plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, ).payload_with(data, mask) - stacked_payload = stack_image_payload_context( + stacked_payload = stack_image_payloads( (payload,), - data[np.newaxis, ...], metadata_mode=ImagePayloadMetadataCompositionMode.STACK, ) @@ -937,9 +932,8 @@ def test_stack_payload_context_composes_single_image_slice_mask_axis(): mask = np.ones_like(data, dtype=bool) payload = ImagePayloadMetadata().payload_with(data, mask) - stacked_payload = stack_image_payload_context( + stacked_payload = stack_image_payloads( (payload,), - stack, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, ) diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index fab4092caf..29c8a6534e 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -19,7 +19,7 @@ AlignedImageSliceContext, ImagePayloadBundleContext, payload_slices_for_alignment, - stack_image_payload_context, + stack_image_payloads, ) from openhcs.core.artifacts import ( ArtifactInputPlan, @@ -404,9 +404,8 @@ def test_workspace_source_projection_carries_exact_aliases_into_stack_provenance for payload in projected_payloads ) - stack = stack_image_payload_context( + stack = stack_image_payloads( projected_payloads, - np.stack(tuple(image_payload_data(payload) for payload in projected_payloads)), metadata_mode=projection.payload_composition_mode(lookups), ) @@ -765,9 +764,8 @@ def test_stack_payload_context_promotes_single_channel_slice_metadata() -> None: ) ) - payload = stack_image_payload_context( + payload = stack_image_payloads( (first, second), - stack, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, ) metadata = image_payload_metadata(payload) diff --git a/tests/unit/test_mixed_carrier_intensity_domain.py b/tests/unit/test_mixed_carrier_intensity_domain.py new file mode 100644 index 0000000000..84df9f3b82 --- /dev/null +++ b/tests/unit/test_mixed_carrier_intensity_domain.py @@ -0,0 +1,233 @@ +"""Current intensity units survive composition, projection and native storage.""" + +from dataclasses import dataclass + +import numpy as np +import pytest + +from openhcs.core.aligned_image_payload import ( + ImagePayloadBundleContext, ImagePayloadStackContext, stack_image_payloads, + ImagePayloadStackComposition, +) +from openhcs.core.image_file_serialization import ImageFileFormat, ImageFileSourceMetadata +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, ImagePayloadMetadataCompositionMode, + ImageUnitIntervalIntensityMetadata, image_payload_data, image_payload_mask, + image_payload_metadata, normalize_image_payload_intensity, +) +from openhcs.core.source_bindings import NamedSourceBinding +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.source_spatial_domain import SourceSpatialDomain + + +def source(data, *, scale=255, channel_axis=None): + return ImagePayloadMetadata( + source_path='/declared/source.tif', source_dtype=str(data.dtype), + intensity_scale=scale, source_channel_axis=channel_axis, + source_spatial_domain=SourceSpatialDomain((3, 5), (2, 3), 0, 'Source'), + source_voxel_spacing=SourceVoxelSpacing((0.5, 0.5)), + ).payload_with(data, np.array([[True, False, True], [True, True, False]])) + + +@pytest.mark.parametrize('reverse', (False, True)) +@pytest.mark.parametrize('color', ('replicated', 'distinct')) +@pytest.mark.parametrize('mode', tuple(ImagePayloadMetadataCompositionMode)) +def test_original_source_binding_composition_and_selected_cp_units(reverse, color, mode): + from openhcs.interop.cellprofiler.image_normalization import normalize_cellprofiler_image_payload + + scalar = np.full((2, 3), 128, dtype=np.uint8) + rgb = np.repeat(scalar[..., None], 3, axis=-1) + if color == 'distinct': + rgb[..., 1] = 32 + rgb[..., 2] = 240 + binding = NamedSourceBinding(alias='Source', load_as_monochrome=True) + inputs = [binding.apply_loaded_payload(source(scalar), None), + binding.apply_loaded_payload(source(rgb, channel_axis=-1), None)] + if reverse: + inputs.reverse() + stacked = stack_image_payloads(inputs, metadata_mode=mode) + for index, original in enumerate(inputs): + selected = image_payload_metadata(stacked).for_leading_source_plane(index).payload_with( + image_payload_data(stacked)[index], image_payload_mask(stacked)[index], + ) + expected = normalize_cellprofiler_image_payload(original) + actual = normalize_cellprofiler_image_payload(selected) + np.testing.assert_allclose(image_payload_data(actual), image_payload_data(expected)) + np.testing.assert_array_equal(image_payload_mask(actual), image_payload_mask(original)) + assert image_payload_metadata(actual).source_image_names == ('Source',) + assert image_payload_metadata(actual).intensity_scale == 255 + assert image_payload_metadata(actual).source_voxel_spacing == image_payload_metadata(original).source_voxel_spacing + actual_domain = image_payload_metadata(actual).source_spatial_domain + original_domain = image_payload_metadata(original).source_spatial_domain + assert actual_domain.origin_yx == original_domain.origin_yx + assert actual_domain.source_shape_yx == original_domain.source_shape_yx + assert actual_domain.fill_value == original_domain.fill_value + + +@pytest.mark.parametrize('reverse', (False, True)) +def test_raw_heterogeneous_declared_scales_survive_integer_float_promotion(reverse): + inputs = [source(np.full((2, 3), 30, dtype=np.uint8), scale=60), + source(np.full((2, 3), 15, dtype=np.float32), scale=30)] + if reverse: + inputs.reverse() + stack = stack_image_payloads(inputs, metadata_mode=ImagePayloadMetadataCompositionMode.STACK) + assert image_payload_metadata(stack).unit_interval_intensity is None + np.testing.assert_allclose(normalize_image_payload_intensity(stack), 0.5) + for index, original in enumerate(inputs): + projected = image_payload_metadata(stack).for_leading_source_plane(index).payload_with( + image_payload_data(stack)[index]) + np.testing.assert_array_equal(normalize_image_payload_intensity(projected), + normalize_image_payload_intensity(original)) + + +def test_normalized_and_remapped_float_pixels_do_not_reenter_source_codes(): + normalized = normalize_image_payload_intensity(source(np.full((2, 3), 128, dtype=np.uint8))) + remapped = image_payload_metadata(normalized).without_unit_interval_intensity_scale().payload_with( + image_payload_data(normalized) * 4 - 1, image_payload_mask(normalized), + ) + for payload in (normalized, remapped): + np.testing.assert_array_equal(normalize_image_payload_intensity(payload), payload) + assert image_payload_metadata(normalize_image_payload_intensity(payload)) == image_payload_metadata(payload) + + +def test_unscaled_analytical_float_uses_no_observed_range_or_guessed_factor(): + pixels = np.array([[-3.5, 800.0], [0.25, 2.0]], dtype=np.float32) + np.testing.assert_array_equal(normalize_image_payload_intensity(pixels), pixels) + + +@pytest.mark.parametrize('scale', (0, -1, float('nan'), float('inf'))) +def test_invalid_declared_scale_is_rejected(scale): + with pytest.raises(ValueError, match='finite and positive'): + normalize_image_payload_intensity(source(np.ones((2, 3), dtype=np.float32), scale=scale)) + + +def test_native_pixel_conversion_resets_current_domain_and_float_roundtrip_preserves_it(tmp_path): + normalized = normalize_image_payload_intensity(source(np.full((2, 3), 128, dtype=np.uint8))) + metadata = image_payload_metadata(normalized) + native = ImageFileSourceMetadata(np.dtype('uint8'), 255).project_image_metadata( + metadata, values_preserved=False, + ) + assert native.unit_interval_intensity is None + np.testing.assert_allclose(normalize_image_payload_intensity( + native.payload_with(np.full((2, 3), 128, dtype=np.uint8))), normalized) + path = tmp_path / 'analytical.npy' + image_format = ImageFileFormat.require_path(path) + image_format.write(path, normalized) + restored = image_format.persisted_metadata(path, normalized) + assert restored.unit_interval_intensity == metadata.unit_interval_intensity + np.testing.assert_array_equal(normalize_image_payload_intensity( + restored.payload_with(image_format.read(path))), normalized) + assert restored.source_spatial_domain == metadata.source_spatial_domain + assert restored.source_voxel_spacing == metadata.source_voxel_spacing + + +@pytest.mark.parametrize('reverse', (False, True)) +def test_independent_normalization_and_composition_capabilities_use_cooperative_mro(reverse): + calls = [] + + class NormalizationAudit: + def normalize_intensity_payload(self, payload, **kwargs): + calls.append('normalization') + return super().normalize_intensity_payload(payload, **kwargs) + + @dataclass + class DeclaredMetadata(NormalizationAudit, ImagePayloadMetadata): + pass + + class PixelAudit: + def compose_unmasked(self, payloads, **kwargs): + calls.append('pixels') + return super().compose_unmasked(payloads, **kwargs) + + class MaskAudit: + def compose_mask(self, data, metadata): + calls.append('mask') + return super().compose_mask(data, metadata) + + capabilities = (MaskAudit, PixelAudit) if reverse else (PixelAudit, MaskAudit) + + class DeclaredComposition(*capabilities, ImagePayloadStackContext): + pass + + raw = DeclaredMetadata(intensity_scale=64, source_dtype='uint8').payload_with( + np.full((2, 3), 32, dtype=np.uint8)) + processed = ImagePayloadMetadata(unit_interval_intensity=ImageUnitIntervalIntensityMetadata()).payload_with( + np.full((2, 3), 0.5, dtype=np.float32)) + result = DeclaredComposition((raw, processed), ImagePayloadMetadataCompositionMode.STACK).compose() + np.testing.assert_array_equal(image_payload_data(result), 0.5) + assert calls == ['normalization', 'pixels', 'mask'] + assert DeclaredMetadata.__mro__[:3] == (DeclaredMetadata, NormalizationAudit, ImagePayloadMetadata) + + +def test_bundle_leaf_cooperates_with_shared_intensity_and_pixel_owners(): + raw = source(np.full((2, 3), 128, dtype=np.uint8)) + normalized = normalize_image_payload_intensity(raw) + result = ImagePayloadBundleContext.from_payloads((raw, normalized)).compose() + np.testing.assert_array_equal(image_payload_data(result)[0], image_payload_data(normalized)) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(raw)) + + +def test_saved_output_context_retains_the_independent_composed_buffer_domain(): + raw = source(np.full((2, 3), 128, dtype=np.uint8)) + normalized = normalize_image_payload_intensity(raw) + inputs = (raw, normalized) + copied = stack_image_payloads(inputs, metadata_mode=ImagePayloadMetadataCompositionMode.STACK) + restored = ImagePayloadStackComposition.with_saved_output_context( + copied, inputs, tuple(image_payload_metadata(value) for value in inputs), + single_output_plane_axis=None, + ) + assert image_payload_data(restored) is image_payload_data(copied) + for index in (0, 1): + selected = image_payload_metadata(restored).for_leading_source_plane(index).payload_with( + image_payload_data(restored)[index]) + np.testing.assert_array_equal(normalize_image_payload_intensity(selected), normalized) + assert image_payload_metadata(selected).has_normalized_intensity + + +def test_common_quantization_requires_every_current_plane_not_only_known_members(): + raw = source(np.full((2, 3), 128, dtype=np.uint8)) + normalized = normalize_image_payload_intensity(raw) + remapped = image_payload_metadata(normalized).without_unit_interval_intensity_scale().payload_with( + np.full((2, 3), 0.123456, dtype=np.float32)) + mixed = stack_image_payloads((raw, remapped), metadata_mode=ImagePayloadMetadataCompositionMode.STACK) + assert image_payload_metadata(mixed).common_unit_interval_intensity_scale() is None + assert image_payload_metadata(mixed).for_leading_source_plane(0).unit_interval_intensity_scale == 255 + + +@pytest.mark.parametrize('normalized', (False, True)) +def test_proof_invalidation_preserves_the_current_numerical_domain(normalized): + from openhcs.processing.backends.cellprofiler.color import ( + color_to_gray_combine_output_metadata, + ) + + raw = source(np.full((2, 3), 128, dtype=np.uint8)) + payload = normalize_image_payload_intensity(raw) if normalized else raw + # Use the original arithmetic output metadata owner, not a test-side marker. + metadata = color_to_gray_combine_output_metadata(payload) + transformed = metadata.payload_with( + image_payload_data(payload).astype(np.float32) * 0.5, + image_payload_mask(payload), + ) + assert metadata.has_normalized_intensity is normalized + assert metadata.unit_interval_intensity_scale is None + expected = image_payload_data(payload) * (0.5 if normalized else 0.5 / 255) + np.testing.assert_allclose(normalize_image_payload_intensity(transformed), expected) + assert metadata.intensity_scale == 255 + + +@pytest.mark.parametrize('normalized', (False, True)) +def test_cellprofiler_registered_image_input_owner_enters_normalized_domain(normalized): + from openhcs.core.artifacts import ArtifactSpec, ImageArtifactType + from openhcs.interop.cellprofiler.runtime.output_recording import CellProfilerOutputRecorder + + raw = source(np.full((2, 3), 128, dtype=np.uint8)) + payload = normalize_image_payload_intensity(raw) if normalized else raw + owner = CellProfilerOutputRecorder.for_artifact_type(ImageArtifactType) + result = owner.runtime_input_value(ArtifactSpec.input('Input', ImageArtifactType), payload) + np.testing.assert_allclose(image_payload_data(result), 128 / 255) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(raw)) + metadata = image_payload_metadata(result) + assert metadata.has_normalized_intensity + assert metadata.unit_interval_intensity_scale == 255 + assert metadata.source_image_names == ('Input',) + assert metadata.without_unit_interval_intensity_scale().has_normalized_intensity diff --git a/tests/unit/test_persisted_image_metadata.py b/tests/unit/test_persisted_image_metadata.py index 07a1ca4606..eb95e7d4a2 100644 --- a/tests/unit/test_persisted_image_metadata.py +++ b/tests/unit/test_persisted_image_metadata.py @@ -109,7 +109,7 @@ def test_saved_format_metadata_is_native_and_does_not_reencode( assert restored.source_provenance == metadata.source_provenance assert restored.plane_axis is None if extension in (".png", ".bmp"): - assert restored.unit_interval_intensity.scale is None + assert restored.unit_interval_intensity is None assert restored.physical_border_edges_yx is None assert restored.mask_defines_border is None else: @@ -489,7 +489,7 @@ def test_absent_native_scale_retains_only_value_preserved_authored_facts(): changed = header.project_image_metadata(metadata, values_preserved=False) assert changed.intensity_scale is None assert changed.source_plane_intensity_scales == (None, None) - assert changed.unit_interval_intensity.scale is None + assert changed.unit_interval_intensity is None @pytest.mark.parametrize("origin", ((0, 0), (7, 11))) @@ -696,7 +696,7 @@ def test_lossy_jpeg_clears_exact_value_proofs_even_for_uint8(tmp_path): image_format.write(path, payload) assert np.any(image_format.read(path) != pixels) restored = image_format.persisted_metadata(path, payload) - assert restored.unit_interval_intensity.scale is None + assert restored.unit_interval_intensity is None assert restored.physical_border_edges_yx is None assert restored.mask_defines_border is None assert restored.source_provenance == metadata.source_provenance @@ -741,7 +741,7 @@ def test_mixed_zarr_batch_preserves_lineage_but_not_quantized_value_proofs(tmp_p ), ) assert restored.source_dtype == "uint8" - assert restored.unit_interval_intensity.scale is None + assert restored.unit_interval_intensity is None assert restored.physical_border_edges_yx is None assert restored.mask_defines_border is None assert restored.source_provenance == metadata.source_provenance diff --git a/tests/unit/test_runtime_values.py b/tests/unit/test_runtime_values.py index 10fa5eb3f1..0e6911c10d 100644 --- a/tests/unit/test_runtime_values.py +++ b/tests/unit/test_runtime_values.py @@ -3463,7 +3463,7 @@ def test_default_image_output_context_preserves_explicit_intensity_proof_invalid ) output = ( ImagePayloadMetadata(source_dtype="float32") - .without_unit_interval_intensity_scale() + .with_unit_interval_intensity_scale(None) .payload_with(np.ones((4, 5), dtype=np.float32)) ) diff --git a/validation/mixed-carrier-intensity-domain-599/run_source.py b/validation/mixed-carrier-intensity-domain-599/run_source.py new file mode 100644 index 0000000000..ee1fca1120 --- /dev/null +++ b/validation/mixed-carrier-intensity-domain-599/run_source.py @@ -0,0 +1,50 @@ +"""Use the original source bootstrap and pytest; no native/catalog initialization.""" + +import hashlib +import json +from pathlib import Path +import sys + +owned = Path(__file__).resolve().parents[2] +root = Path('/home/ts/wt/openhcs-issue-batch-20260929/engineering599') +source = owned +diagnostic = '--original-diagnostic' in sys.argv +if diagnostic: + source = root / 'original-main' + sys.argv.remove('--original-diagnostic') +sys.path.insert(0, '/home/ts/code/projects/openhcs/.venv/lib/python3.12/site-packages') +sys.path.insert(0, str(source)) +import openhcs +from openhcs._source_dependencies import ensure_source_checkout_external_paths +ensure_source_checkout_external_paths(root / 'source-runtime') + +import numpy as np +import openhcs.core.runtime_image_values as values +from openhcs.core.aligned_image_payload import stack_image_payloads +assert Path(values.__file__).is_relative_to(source) +print(json.dumps({'source': str(source), 'metadata_module': values.__file__, + 'sha256': hashlib.sha256(Path(values.__file__).read_bytes()).hexdigest(), + 'mode': 'original source; no viewer/server/catalog/installed acceptance'}), flush=True) + +if diagnostic: + scalar = np.full((2, 3), 128, dtype=np.uint8) + raw = values.ImagePayloadMetadata.for_array(scalar).payload_with(scalar) + normalized = values.normalize_image_payload_intensity(raw) + rgb_mono = values.ImagePayloadMetadata( + source_dtype='uint8', + unit_interval_intensity=values.ImageUnitIntervalIntensityMetadata(), + ).payload_with(np.full((2, 3), 128 / 255, dtype=np.float32)) + mixed = stack_image_payloads((raw, rgb_mono), + metadata_mode=values.ImagePayloadMetadataCompositionMode.BUNDLE) + selected = values.image_payload_metadata(mixed).for_leading_source_plane(0).payload_with( + values.image_payload_data(mixed)[0]) + actual = values.normalize_image_payload_intensity(selected) + print(json.dumps({'independent': np.asarray(normalized).tolist(), + 'mixed_then_selected': np.asarray(actual).tolist(), + 'mixed_dtype': str(np.asarray(mixed).dtype), + 'equivalent': bool(np.allclose(actual, normalized))}), flush=True) + raise SystemExit(0) + +import pytest +raise SystemExit(pytest.main(['--noconftest', '--import-mode=importlib', + '-p', 'no:cacheprovider', '-o', 'addopts=', '-q', *sys.argv[1:]])) diff --git a/validation/mixed-carrier-intensity-domain-599/source_family.py b/validation/mixed-carrier-intensity-domain-599/source_family.py new file mode 100644 index 0000000000..fa5a1510df --- /dev/null +++ b/validation/mixed-carrier-intensity-domain-599/source_family.py @@ -0,0 +1,96 @@ +"""Read exact Git source through the existing audit Package parser; no product imports.""" + +import ast +import argparse +import hashlib +import json +from pathlib import Path +import sys + +sys.path.insert(0, "/home/ts/.codex/skills/refactor-audit/scripts") +from audit.findings import Package, ParsedModule +from audit.repository import Repository + + +class SourceScopeRepository(Repository): + """Reuse the single-file scope adaptation qualified in source-family04.""" + + def python_files(self, revision, root): + if root.endswith(".py"): + return (root,) + return super().python_files(revision, root) + + +checkout = Path(__file__).resolve().parents[2] +repo = SourceScopeRepository(checkout) +revision = repo.git("rev-parse", "HEAD").strip() +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--dependency-root", action="append", default=[]) +parser.add_argument("--only-dependencies", action="store_true") +arguments = parser.parse_args() +source_roots = dict(value.split("=", 1) for value in arguments.dependency_root) +dependencies = [] +for line in repo.git("ls-tree", "-r", revision).splitlines(): + descriptor, path = line.split("\t", 1) + mode, _, object_id = descriptor.split() + if mode == "160000": + if not arguments.only_dependencies or path in source_roots: + dependencies.append((Path(source_roots.get(path, + str(Path("/home/ts/code/projects/openhcs") / path))), object_id, None)) + +terms = ( + "ImagePayloadMetadata", "ImageUnitIntervalIntensityMetadata", "intensity_scale", + "source_dtype", "unit_interval_intensity", "normalize_image_payload_intensity", + "normalize_cellprofiler_image_payload", "apply_loaded_payload", "rgb2gray", + "ImagePayloadStackComposition", "stack_runtime_slices", "ImageFileSourceMetadata", +) +failures = [] +production = () if arguments.only_dependencies else tuple( + (checkout, revision, root) for root in ('openhcs', 'scripts', 'benchmark', 'tests') +) +for location, version, prefix in (*production, *dependencies): + source_repo = SourceScopeRepository(location) + try: + listing = source_repo.git("ls-tree", "-r", "--name-only", version).splitlines() + paths = tuple(path for path in listing if path.endswith(".py") and ( + prefix is None or path.startswith(prefix + "/") + )) + scopes = sorted({str(Path(*Path(path).parts[:2])) if prefix else + Path(path).parts[0] for path in paths}) + total = selected = 0 + for scope in scopes: + package = Package.load(source_repo, version, scope) + failures.extend((str(location), path) for path in package.unparsed) + total += len(package.modules) + for module in package.modules: + if any(term in module.text for term in terms): + selected += 1 + print(json.dumps({"root": str(location), "revision": version, + "path": module.path, + "sha256": hashlib.sha256(module.text.encode()).hexdigest(), + "ast": ast.dump(module.tree, include_attributes=True)})) + del package + print(json.dumps({"root": str(location), "revision": version, + "parsed": total, "selected": selected}), flush=True) + except Exception as error: + failures.append((str(location), repr(error))) + print(json.dumps({"root": str(location), "revision": version, + "error": repr(error)}), flush=True) + +for file in (() if arguments.only_dependencies else ( + "/home/ts/code/projects/openhcs/.venv/lib/python3.12/site-packages/numpy/_core/shape_base.py", + "/home/ts/code/projects/openhcs/.venv/lib/python3.12/site-packages/skimage/color/colorconv.py", + "/home/ts/code/projects/openhcs/.venv/lib/python3.12/site-packages/skimage/util/dtype.py", +)): + path = Path(file) + try: + text = path.read_text() + module = ParsedModule(file, text, ast.parse(text, file), file) + print(json.dumps({"source_api": file, + "sha256": hashlib.sha256(text.encode()).hexdigest(), + "ast": ast.dump(module.tree, include_attributes=True)})) + except Exception as error: + failures.append((file, repr(error))) +print(json.dumps({"failures": failures, + "limit": "AST source identity/closure; no imported product, behavioral or R1 proof"}), flush=True) +raise SystemExit(bool(failures))