From 2cbdceff0dbad0a619a18ee2d7282b85dc430ee4 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 13:55:34 -0400 Subject: [PATCH 001/125] Fuse source-plane metadata projection through nominal transformations --- openhcs/core/runtime_image_values.py | 181 +++++++++++++----- openhcs/core/runtime_measurements.py | 5 +- openhcs/core/runtime_object_labels.py | 8 +- openhcs/core/runtime_relationships.py | 4 +- openhcs/core/runtime_spatial_graph.py | 4 +- openhcs/core/source_image_provenance.py | 8 +- .../test_image_payload_projection_algebra.py | 154 +++++++++++++++ ..._source_provenance_constructor_defaults.py | 99 ++++++++++ 8 files changed, 397 insertions(+), 66 deletions(-) create mode 100644 tests/unit/test_image_payload_projection_algebra.py create mode 100644 tests/unit/test_source_provenance_constructor_defaults.py diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index a29f7af397..e7ae62ffb7 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -250,15 +250,21 @@ def for_array_payload( ) def __post_init__(self, *source_provenance_values: object) -> None: - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.source_voxel_spacing = self.source_voxel_spacing.with_missing_from( SourceVoxelSpacing.from_source_metadata(self.source_component_metadata) ) self.normalize_source_provenance_fields() self.normalize_source_spatial_domain_fields() self.normalize_source_voxel_spacing_fields() + self.validate_source_channel_axis() + if self.plane_axis is not None: + self.plane_axis = RuntimePlaneAxis( + self.plane_axis, + ) + + def validate_source_channel_axis(self) -> None: + """Validate the authored channel declaration before transforming axes.""" if self.source_channel_axis is not None and ( not isinstance(self.source_channel_axis, int) or isinstance(self.source_channel_axis, bool) @@ -266,10 +272,6 @@ def __post_init__(self, *source_provenance_values: object) -> None: raise TypeError( "ImagePayloadMetadata.source_channel_axis must be int or None." ) - if self.plane_axis is not None: - self.plane_axis = RuntimePlaneAxis( - self.plane_axis, - ) @property def has_values(self) -> bool: @@ -649,37 +651,11 @@ def without_source_channel_axis(self) -> "ImagePayloadMetadata": def for_leading_source_plane(self, plane_index: int) -> "ImagePayloadMetadata": """Project metadata after explicitly removing its leading plane axis.""" - if self.plane_axis is None: - raise ValueError( - "Leading source-plane projection requires a declared plane axis." - ) - return self.for_source_plane(plane_index).without_leading_plane_axis() + return LeadingSourcePlaneMetadataProjection(self, plane_index).project() def without_leading_plane_axis(self) -> "ImagePayloadMetadata": """Return metadata after an explicitly declared leading axis is removed.""" - if self.plane_axis is None: - raise ValueError("Image metadata has no leading plane axis to remove.") - source_channel_axis = self.source_channel_axis - if source_channel_axis == 0: - raise ValueError( - "Image metadata cannot declare the same leading axis as both " - "plane and channel." - ) - if source_channel_axis is not None and source_channel_axis > 0: - source_channel_axis -= 1 - return self.with_source_provenance( - self.source_provenance.with_runtime_planes_as_contributors() - ).replace_fields( - plane_axis=None, - source_channel_axis=source_channel_axis, - source_plane_intensity_scales=(), - source_plane_dtypes=(), - unit_interval_intensity=( - None - if self.unit_interval_intensity is None - else self.unit_interval_intensity.without_source_planes() - ), - ) + return LeadingPlaneAxisMetadataProjection(self).project() def collapse_leading_plane_axis(self) -> "ImagePayloadMetadata": """Return scalar metadata after reducing every plane of the leading axis.""" @@ -786,20 +762,7 @@ def payload_with(self, data: Any, mask: Any | None = None) -> Any: def for_source_plane(self, plane_index: int) -> "ImagePayloadMetadata": """Return metadata for one source plane sliced from a stacked payload.""" - source_provenance = self.source_provenance.for_source_plane(plane_index) - return self.replace_fields( - intensity_scale=self.intensity_scale_for_source_plane(plane_index), - source_dtype=_tuple_value(self.source_plane_dtypes, plane_index) - or self.source_dtype, - source_provenance=source_provenance, - unit_interval_intensity=( - None - if self.unit_interval_intensity is None - else self.unit_interval_intensity.for_source_plane(plane_index) - ), - source_plane_intensity_scales=(), - source_plane_dtypes=(), - ) + return SourcePlaneImageMetadataProjection(self, plane_index).project() def for_source_planes( self, @@ -1447,6 +1410,126 @@ def with_image_payload_data( return resolved_metadata.payload_with(data, resolved_mask) +@dataclass(frozen=True, slots=True) +class ImageMetadataProjection(ABC): + """Construct one metadata result from nominal source and axis transformations.""" + + metadata: ImagePayloadMetadata + + def project(self) -> ImagePayloadMetadata: + return self.metadata.replace_fields(**self.projected_fields()) + + @property + def projected_source_provenance(self) -> SourceImageProvenance: + return self.metadata.source_provenance + + @property + def projected_unit_interval_intensity( + self, + ) -> ImageUnitIntervalIntensityMetadata | None: + return self.metadata.unit_interval_intensity + + @abstractmethod + def projected_fields(self) -> dict[str, Any]: + """Derive constructor instructions, never an independently stored fact set.""" + return dict( + source_provenance=self.projected_source_provenance, + unit_interval_intensity=self.projected_unit_interval_intensity, + ) + + +@dataclass(frozen=True, slots=True) +class SourcePlaneImageMetadataProjection(ImageMetadataProjection): + """Select scalar source-image facts without removing an array axis.""" + + plane_index: int + + @property + def projected_source_provenance(self) -> SourceImageProvenance: + return super( + SourcePlaneImageMetadataProjection, self + ).projected_source_provenance.for_source_plane(self.plane_index) + + @property + def projected_unit_interval_intensity( + self, + ) -> ImageUnitIntervalIntensityMetadata | None: + intensity = super( + SourcePlaneImageMetadataProjection, self + ).projected_unit_interval_intensity + return ( + None if intensity is None else intensity.for_source_plane(self.plane_index) + ) + + def projected_fields(self) -> dict[str, Any]: + changes = super(SourcePlaneImageMetadataProjection, self).projected_fields() + changes.update( + intensity_scale=self.metadata.intensity_scale_for_source_plane( + self.plane_index + ), + source_dtype=_tuple_value( + self.metadata.source_plane_dtypes, self.plane_index + ) + or self.metadata.source_dtype, + source_plane_intensity_scales=(), + source_plane_dtypes=(), + ) + return changes + + +class LeadingPlaneAxisMetadataProjection(ImageMetadataProjection): + """Remove a declared leading axis while preserving its source contributors.""" + + @property + def projected_source_provenance(self) -> SourceImageProvenance: + return super().projected_source_provenance.with_runtime_planes_as_contributors() + + @property + def projected_unit_interval_intensity( + self, + ) -> ImageUnitIntervalIntensityMetadata | None: + intensity = super().projected_unit_interval_intensity + return None if intensity is None else intensity.without_source_planes() + + def projected_fields(self) -> dict[str, Any]: + if self.metadata.plane_axis is None: + raise ValueError("Image metadata has no leading plane axis to remove.") + source_channel_axis = self.metadata.source_channel_axis + if source_channel_axis == 0: + raise ValueError( + "Image metadata cannot declare the same leading axis as both " + "plane and channel." + ) + self.metadata.validate_source_channel_axis() + RuntimePlaneAxis(self.metadata.plane_axis) + if source_channel_axis is not None and source_channel_axis > 0: + source_channel_axis -= 1 + changes = super().projected_fields() + changes.update( + plane_axis=None, + source_channel_axis=source_channel_axis, + source_plane_intensity_scales=(), + source_plane_dtypes=(), + ) + return changes + + +class LeadingSourcePlaneMetadataProjection( + LeadingPlaneAxisMetadataProjection, + SourcePlaneImageMetadataProjection, +): + """Compose source selection and axis removal before constructing metadata.""" + + def projected_fields(self) -> dict[str, Any]: + if self.metadata.plane_axis is None: + raise ValueError( + "Leading source-plane projection requires a declared plane axis." + ) + self.metadata.validate_source_channel_axis() + RuntimePlaneAxis(self.metadata.plane_axis) + return super().projected_fields() + + @dataclass(frozen=True, slots=True) class ImagePayloadSliceProjector: """Project payload context from a parent image into one child image slice.""" diff --git a/openhcs/core/runtime_measurements.py b/openhcs/core/runtime_measurements.py index b0c613097d..a213bd62d8 100644 --- a/openhcs/core/runtime_measurements.py +++ b/openhcs/core/runtime_measurements.py @@ -15,7 +15,6 @@ MeasurementObjectRowIdentity, ) from openhcs.core.source_image_provenance import ( - SourceImageProvenance, SourceImageProvenanceFields, ) @@ -75,9 +74,7 @@ def iter_row_mappings(self) -> Iterable[Mapping[str, object]]: return self.rows.iter_row_mappings() def __post_init__(self, *source_provenance_values: object) -> None: - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_source_provenance_fields() self.validate_artifact_name() if self.source_image_name == "": diff --git a/openhcs/core/runtime_object_labels.py b/openhcs/core/runtime_object_labels.py index 69e07dbd46..056dc95c89 100644 --- a/openhcs/core/runtime_object_labels.py +++ b/openhcs/core/runtime_object_labels.py @@ -928,9 +928,7 @@ class ObjectLabelPayload(ObjectLabelValue): def __post_init__(self, *source_provenance_values: object) -> None: self.validate_object_label_variants() - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_object_label_metadata("ObjectLabelPayload") @@ -994,9 +992,7 @@ def from_payload( def __post_init__(self, *source_provenance_values: object) -> None: self.validate_object_label_variants() - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.validate_artifact_name() if self.source_image_name == "": raise ValueError("ObjectLabelSet.source_image_name cannot be empty.") diff --git a/openhcs/core/runtime_relationships.py b/openhcs/core/runtime_relationships.py index 4abca79cfa..1d542b67e1 100644 --- a/openhcs/core/runtime_relationships.py +++ b/openhcs/core/runtime_relationships.py @@ -218,9 +218,7 @@ def from_payload( def __post_init__(self, *source_provenance_values: object) -> None: self.validate_artifact_name() - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_source_provenance_fields() if not isinstance(self.declaration, ObjectRelationshipDeclaration): raise TypeError( diff --git a/openhcs/core/runtime_spatial_graph.py b/openhcs/core/runtime_spatial_graph.py index f12cc3abaf..db26f65412 100644 --- a/openhcs/core/runtime_spatial_graph.py +++ b/openhcs/core/runtime_spatial_graph.py @@ -201,9 +201,7 @@ class SpatialGraph(SourceImageProvenanceFields, NamedArtifactPayload): source_plane_index: int | None = None def __post_init__(self, *source_provenance_values: object) -> None: - self.absorb_explicit_source_provenance( - SourceImageProvenance.from_init_values(source_provenance_values) - ) + self.absorb_explicit_source_provenance(source_provenance_values) self.normalize_source_provenance_fields() self.validate_artifact_name() nodes = tuple(self.nodes) diff --git a/openhcs/core/source_image_provenance.py b/openhcs/core/source_image_provenance.py index 71ff6ef174..3b16137a42 100644 --- a/openhcs/core/source_image_provenance.py +++ b/openhcs/core/source_image_provenance.py @@ -1775,8 +1775,14 @@ class SourceImageProvenanceFields: def absorb_explicit_source_provenance( self, - explicit: SourceImageProvenance, + values: SourceProvenanceInitValues, ) -> None: + """Decode authored aliases once, leaving absent constructor facts absent.""" + if all(value is None for value in values[:-1]): + names = values[-1] + if type(names) is tuple and not names: + return + explicit = SourceImageProvenance.from_init_values(values) if not explicit.has_values: return self.source_provenance = explicit.with_missing_from(self.source_provenance) diff --git a/tests/unit/test_image_payload_projection_algebra.py b/tests/unit/test_image_payload_projection_algebra.py new file mode 100644 index 0000000000..ef898d9870 --- /dev/null +++ b/tests/unit/test_image_payload_projection_algebra.py @@ -0,0 +1,154 @@ +"""Physical source-plane projection preserves composition and mutable isolation.""" + +from dataclasses import fields + +import numpy as np +import pytest + +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + ImagePayloadSliceProjector, + ImageUnitIntervalIntensityMetadata, + image_payload_mask, + image_payload_metadata, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_image_provenance import ( + RuntimeSourceImageProvenancePlane, + SourceImageIdentity, + SourceImageProvenanceContributor, + SourceImageProvenancePlanes, +) + + +def _metadata(axis, channel_axis): + return ImagePayloadMetadata( + source_path="/tmp/parent.tif", + source_component_metadata={"well": "A01", "extension": ".tif"}, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/tmp/first.tif", "/tmp/second.tif"), + component_metadata=({"site": "1"}, {"site": "2"}), + ), + intensity_scale=255.0, + source_dtype="uint8", + source_plane_intensity_scales=(255.0, 65535.0), + source_plane_dtypes=("uint8", "uint16"), + unit_interval_intensity=ImageUnitIntervalIntensityMetadata( + scale=255, source_plane_scales=(255, 65535) + ), + source_image_names=("DNA", "RNA"), + source_channel_axis=channel_axis, + plane_axis=axis, + ) + + +@pytest.mark.parametrize("axis", tuple(RuntimePlaneAxis)) +@pytest.mark.parametrize("channel_axis", (None, 1, 3, -1)) +@pytest.mark.parametrize("plane", (0, 1)) +def test_physical_projection_agrees_with_separate_semantic_operations( + axis, channel_axis, plane +): + metadata = _metadata(axis, channel_axis) + sequential = metadata.for_source_plane(plane).without_leading_plane_axis() + projected = metadata.for_leading_source_plane(plane) + for member in fields(ImagePayloadMetadata): + assert getattr(projected, member.name) == getattr(sequential, member.name) + assert projected.source_path == ("/tmp/first.tif", "/tmp/second.tif")[plane] + assert projected.source_component_metadata["well"] == "A01" + assert projected.source_component_metadata["site"] == str(plane + 1) + assert projected.unit_interval_intensity.scale == (255, 65535)[plane] + assert projected.source_plane_intensity_scales == () + assert projected.source_plane_dtypes == () + assert projected.plane_axis is None + projected.source_provenance.source_identity.path = "/tmp/changed.tif" + assert metadata.source_path == "/tmp/parent.tif" + assert metadata.source_image_provenance_planes.paths[plane] != "/tmp/changed.tif" + + +def test_physical_projection_constructs_only_the_final_metadata(monkeypatch): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + calls = [] + original = ImagePayloadMetadata.__post_init__ + + def observe(self, *values): + calls.append(self) + original(self, *values) + + monkeypatch.setattr(ImagePayloadMetadata, "__post_init__", observe) + projected = metadata.for_leading_source_plane(1) + assert calls == [projected] + + +def test_physical_projection_rejects_leading_channel_axis(): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, 0) + with pytest.raises(ValueError, match="both plane and channel"): + metadata.for_leading_source_plane(1) + + +def test_physical_projection_requires_declared_plane_axis(): + metadata = _metadata(None, None) + with pytest.raises(ValueError, match="declared plane axis"): + metadata.for_leading_source_plane(1) + + +@pytest.mark.parametrize("channel_axis", (False, True, "1", 1.5)) +def test_physical_projection_preserves_invalid_mutated_channel_error(channel_axis): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + metadata.source_channel_axis = channel_axis + with pytest.raises(TypeError, match="must be int or None"): + metadata.for_source_plane(1).without_leading_plane_axis() + with pytest.raises(TypeError, match="must be int or None"): + metadata.for_leading_source_plane(1) + + +def test_physical_projection_preserves_invalid_mutated_plane_error(): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + metadata.plane_axis = "invalid-axis" + with pytest.raises(ValueError): + metadata.for_source_plane(1).without_leading_plane_axis() + with pytest.raises(ValueError): + metadata.for_leading_source_plane(1) + + +def test_projection_preserves_selected_spacing_and_nested_contributors(): + contributor = SourceImageProvenanceContributor( + SourceImageIdentity("/tmp/contributor.tif", {"site": "3"}), + source_image_name="Original", + ) + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_image_provenance_planes=SourceImageProvenancePlanes( + ( + RuntimeSourceImageProvenancePlane( + SourceImageIdentity( + "/tmp/selected.tif", + {"OpenHCSSourceVoxelSpacingZYX": "4,2,1"}, + ), + contributors=(contributor,), + source_image_name="DNA", + ), + ) + ), + ) + sequential = metadata.for_source_plane(0).without_leading_plane_axis() + projected = metadata.for_leading_source_plane(0) + for member in fields(ImagePayloadMetadata): + assert getattr(projected, member.name) == getattr(sequential, member.name) + assert projected.source_image_provenance_planes.planes == (contributor,) + + +def test_projection_observes_mask_and_metadata_mutation(): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + mask = np.ones((2, 3, 4), dtype=bool) + projector = ImagePayloadSliceProjector(mask, metadata) + data = np.zeros((3, 4)) + first = projector.payload_for_slice(data, 1) + assert image_payload_metadata(first).intensity_scale == 65535.0 + metadata.source_plane_intensity_scales = (1.0, 2.0) + mask[1, 0, 0] = False + second = projector.payload_for_slice(data, 1) + assert image_payload_metadata(second).intensity_scale == 2.0 + assert not image_payload_mask(second)[0, 0] + mask.resize((2, 2, 6), refcheck=False) + with pytest.raises(ValueError, match="cannot be projected"): + projector.payload_for_slice(data, 1) diff --git a/tests/unit/test_source_provenance_constructor_defaults.py b/tests/unit/test_source_provenance_constructor_defaults.py new file mode 100644 index 0000000000..f25eaa478c --- /dev/null +++ b/tests/unit/test_source_provenance_constructor_defaults.py @@ -0,0 +1,99 @@ +"""All source-metadata carriers decode authored aliases at the same boundary.""" + +from functools import partial + +import numpy as np +import pytest + +from openhcs.core.artifacts import ArtifactSpec, ObjectLabelsArtifactType +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_measurements import ( + MeasurementScope, + MeasurementSubject, + MeasurementTable, +) +from openhcs.core.runtime_object_labels import ( + ObjectLabelPayload, + ObjectLabelSet, + ObjectLabelVariantData, +) +from openhcs.core.runtime_relationships import ( + DirectedObjectRelationshipPayload, + ObjectRelationship, + ObjectRelationshipDeclaration, +) +from openhcs.core.runtime_spatial_graph import SpatialGraph +from openhcs.core.runtime_tabular_values import ColumnarRows +from openhcs.core.source_image_provenance import SourceImageProvenance + + +class EmptyRows(ColumnarRows): + columns = {} + fields = () + + +def _carriers(): + labels = ObjectLabelVariantData(labels=np.zeros((2, 3), dtype=np.int32)) + relationship = ObjectRelationshipDeclaration.parent_child( + source=ArtifactSpec.output("Cells", ObjectLabelsArtifactType).ref(), + target=ArtifactSpec.output("Nuclei", ObjectLabelsArtifactType).ref(), + producer_module_number=1, + ) + return ( + ImagePayloadMetadata, + partial(ObjectLabelPayload, variant_data=labels), + partial(ObjectLabelSet, name="Cells", variant_data=labels), + partial( + MeasurementTable, + name="Intensity", + rows=EmptyRows(), + subject=MeasurementSubject(MeasurementScope.IMAGE, "DNA"), + ), + partial(SpatialGraph, name="Graph", nodes=(), edges=()), + partial( + ObjectRelationship, + name="ParentChild", + declaration=relationship, + payload=DirectedObjectRelationshipPayload(source_ids=(), target_ids=()), + ), + ) + + +@pytest.mark.parametrize("construct", _carriers()) +def test_absent_aliases_do_not_create_empty_provenance(construct, monkeypatch): + supplied = SourceImageProvenance( + source_path="/tmp/source.tif", source_component_metadata={"well": "A01"} + ) + + def unexpected(cls, values): + pytest.fail("Absent source aliases must not allocate a decoded value.") + + monkeypatch.setattr( + SourceImageProvenance, "from_init_values", classmethod(unexpected) + ) + carrier = construct(source_provenance=supplied) + assert carrier.source_path == "/tmp/source.tif" + assert carrier.source_component_metadata["well"] == "A01" + assert carrier.source_provenance is not supplied + carrier.source_provenance.source_identity.path = "/tmp/changed.tif" + assert supplied.source_path == "/tmp/source.tif" + + +@pytest.mark.parametrize("construct", _carriers()) +def test_explicit_empty_metadata_and_aliases_still_decode(construct, monkeypatch): + calls = [] + original = SourceImageProvenance.from_init_values + + def observe(cls, values): + calls.append(values) + return original(values) + + monkeypatch.setattr(SourceImageProvenance, "from_init_values", classmethod(observe)) + empty = construct(source_component_metadata={}) + assert calls == [(None, {}, None, ())] + assert empty.source_component_metadata is not None + assert dict(empty.source_component_metadata) == {} + authored = construct(source_path="/tmp/authored.tif", source_image_names=("DNA",)) + assert authored.source_path == "/tmp/authored.tif" + assert authored.source_image_names == ("DNA",) + assert len(calls) == 2 From f0c1207b673d651a9d5210a024340b7bebac0689 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 13:57:07 -0400 Subject: [PATCH 002/125] Derive absent source aliases from declared constructor defaults --- openhcs/core/source_image_provenance.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openhcs/core/source_image_provenance.py b/openhcs/core/source_image_provenance.py index 3b16137a42..519003cbb0 100644 --- a/openhcs/core/source_image_provenance.py +++ b/openhcs/core/source_image_provenance.py @@ -1778,10 +1778,13 @@ def absorb_explicit_source_provenance( values: SourceProvenanceInitValues, ) -> None: """Decode authored aliases once, leaving absent constructor facts absent.""" - if all(value is None for value in values[:-1]): - names = values[-1] - if type(names) is tuple and not names: - return + if all( + value is default + for value, default in zip( + values, SourceImageProvenance.__init__.__defaults__, strict=True + ) + ): + return explicit = SourceImageProvenance.from_init_values(values) if not explicit.has_values: return From 33aa3fc04a4657411288d8a5640ddf345c89fde8 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:00:15 -0400 Subject: [PATCH 003/125] Keep required plane validation on metadata authority --- openhcs/core/runtime_image_values.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index e7ae62ffb7..0053bcf5fa 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -273,6 +273,12 @@ def validate_source_channel_axis(self) -> None: "ImagePayloadMetadata.source_channel_axis must be int or None." ) + def require_leading_plane_axis(self, message: str) -> RuntimePlaneAxis: + """Validate a required plane declaration at its metadata authority.""" + if self.plane_axis is None: + raise ValueError(message) + return RuntimePlaneAxis(self.plane_axis) + @property def has_values(self) -> bool: """Return whether this metadata carries any semantic image facts.""" @@ -1492,8 +1498,9 @@ def projected_unit_interval_intensity( return None if intensity is None else intensity.without_source_planes() def projected_fields(self) -> dict[str, Any]: - if self.metadata.plane_axis is None: - raise ValueError("Image metadata has no leading plane axis to remove.") + self.metadata.require_leading_plane_axis( + "Image metadata has no leading plane axis to remove." + ) source_channel_axis = self.metadata.source_channel_axis if source_channel_axis == 0: raise ValueError( @@ -1501,7 +1508,6 @@ def projected_fields(self) -> dict[str, Any]: "plane and channel." ) self.metadata.validate_source_channel_axis() - RuntimePlaneAxis(self.metadata.plane_axis) if source_channel_axis is not None and source_channel_axis > 0: source_channel_axis -= 1 changes = super().projected_fields() @@ -1521,12 +1527,10 @@ class LeadingSourcePlaneMetadataProjection( """Compose source selection and axis removal before constructing metadata.""" def projected_fields(self) -> dict[str, Any]: - if self.metadata.plane_axis is None: - raise ValueError( - "Leading source-plane projection requires a declared plane axis." - ) + self.metadata.require_leading_plane_axis( + "Leading source-plane projection requires a declared plane axis." + ) self.metadata.validate_source_channel_axis() - RuntimePlaneAxis(self.metadata.plane_axis) return super().projected_fields() From 9c070354c3b5e3b6a4f8fe41127dfdaee79e22ee Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:09:36 -0400 Subject: [PATCH 004/125] Share actual materialization outputs with publication and observations (#384) --- .../materialization_plumbing_384_20261001.rst | 90 +++++++ .../orchestrator/compiled_plate_execution.py | 31 ++- openhcs/core/orchestrator/worker_execution.py | 70 +++--- openhcs/core/runtime_execution_validation.py | 14 +- openhcs/core/runtime_exports.py | 21 -- openhcs/core/steps/abstract.py | 24 +- .../function_artifact_materialization.py | 119 ++++----- openhcs/core/steps/function_outputs.py | 150 ++++++----- .../processing/materialization/__init__.py | 6 + openhcs/processing/materialization/core.py | 152 +++++++++--- openhcs/runtime/zmq_execution_observation.py | 9 +- openhcs/runtime/zmq_execution_server.py | 12 +- .../test_function_artifact_materialization.py | 145 ++++++----- tests/unit/test_function_outputs.py | 135 +++++++--- .../test_materialization_batch_outcomes.py | 232 ++++++++++++++++++ .../unit/test_runtime_execution_validation.py | 63 +++-- tests/unit/test_zmq_execution_server.py | 33 +-- tests/unit/test_zmq_runtime_outcome_export.py | 3 + 18 files changed, 930 insertions(+), 379 deletions(-) create mode 100644 docs/validation/materialization_plumbing_384_20261001.rst create mode 100644 tests/unit/test_materialization_batch_outcomes.py diff --git a/docs/validation/materialization_plumbing_384_20261001.rst b/docs/validation/materialization_plumbing_384_20261001.rst new file mode 100644 index 0000000000..a0dc46bcbf --- /dev/null +++ b/docs/validation/materialization_plumbing_384_20261001.rst @@ -0,0 +1,90 @@ +One materialization batch for saving, publication and observation +=============================================================== + +This candidate addresses the generic plumbing tracked in issue #384. It is not +an accepted end-to-end performance result. The separate light owner trace bounded +eight repeated renders at approximately 0.297 seconds and metadata finalization +at 0.896 seconds; removing repeated rendering alone cannot close the generic +runtime gap. This is a structural dependency of the broader runtime boundary work. + +Ownership and production flow +----------------------------- + +``MaterializationBatch`` owns one writer-group rendering and saving algorithm; +its primary path is a derived view. ``SavedMaterializationOutputs`` owns only +completed backend outputs. ``MaterializedRuntimeArtifact`` inherits that shared +backend projection and adds the original reduced artifact binding and observation +behavior. Backend-indexed mappings contain actual destination values, not behavior +handlers. Existing WriterSpec declarations continue to determine format-specific +writing. No new runtime ledger, context cache or identity memoization was added. + +``BackendSaver`` reports only payloads accepted by each backend after successful +save calls. Failed saves raise without returning successful outcomes. Artifact +finalization passes those exact outputs to the existing nominal OutputTarget +family. Transient publication targets own the outcomes until publication finishes; +they are excluded from target equality and hashing. Image directories and source +projections consume these outputs without rendering the mutable logical artifact +again. Rendered pixel content is not carried into worker observation transport. + +The already declared ``StepExecutionObservation`` now carries actual persistent +locations and declared export paths from finalization. Ordinary worker progress +consumes returned locations; worker result observations accumulate returned paths +before image resources are released. Plate-scoped execution uses the same actual +materialization handoff. Existing ``CompiledPlateExecutionExtras`` carries its +RuntimeExecutionObservation; ``CompiledPlateExecutionResults.runtime_observations`` +is a derived collection of worker and parent observations. Both ZMQ value and +outcome exports consume that collection. RuntimeExportObservation no longer +reconstructs output paths from compiled contexts and mutable logical values. + +Terminal materializations persist without external-export participation; +streaming-only specs report no persistent destinations. Measurement reducers can +produce synthetic records absent from RuntimeValueStore; outcomes remain bound to +the record actually reduced and materialized. Main-flow manifests retain their +independent inheritance and semantic-address contracts. + +Removed code and intentionally retained consumers +------------------------------------------------ + +Removed ``_materialization_output_groups`` after moving its shared algorithm to +MaterializationBatch.render. Removed the tests-only materialized and observed +materialized path helpers, the obsolete current-runtime export-path helper, the +single-call observed-materialization wrapper, and +RuntimeExportObservation.from_execution_contexts. Routing tests now intercept +batch preparation; physical/export tests pass actual outcome records explicitly. + +Historical debug reuse still has an explicit prediction surface named +``preview_reused_*``. It does not create a new-save StepExecutionObservation. +Snapshot-owned physical receipts remain OPEN for that separate debug lifecycle; +these prediction functions are not an ordinary execution fallback. Historical +viewer expectations retain their real declaration-based rendering consumer. +Analysis consolidation retains its real CSV-content derivation consumer; it is a +separate reduction, not a replacement authority for observed export paths. + +Evidence and limits +------------------- + +Original source: 10f7e7075b412ccea8959f2c4e1aa818455149fa. NRA original syntax/class +census covers 703 modules, 5157 original ClassDefs, 5145 canonical projections and +12 retained OPEN syntax rows. The bounded ownership decision and complete census +are retained in the external benchmark evidence directory. The authored lifecycle, +new return values and argument migration exceed declaration-movement DSL proof +scope; no native behavioral equivalence proof is asserted. + +The first consumer gate retained 154 passing and 33 failing tests. Routing spies +intercepted the removed production boundary; metadata fixtures omitted the actual +output handoff. Their assertions were preserved while moving those fixtures to the +new boundary. The next six-module gate passed all 197 tests. A later parent gate +passed 264 tests with three new worker-fixture failures: the fixture failed the +existing DebugExecutionContext nominal boundary. It was replaced with a genuine +ProcessingContext subclass; the production boundary was not weakened. + +The final eleven-module gate passes 267 tests in 5.12 seconds. It includes actual +backend acceptance and failed-save controls, exact output-object reuse after +logical input mutation, rejection of re-rendering during ordinary worker progress +and export, all three runtime observation modes, pickle transport after resource +release, exact combined worker/parent export paths, materialization formats, +metadata publication/reconciliation, runtime/viewer validation and ZMQ consumers. + +R0/R1 source guard qualification, representative ordinary unprofiled timing and +native numerical parity remain required before promotion. No benchmark ratios or +speedup claim are attached to this receipt. diff --git a/openhcs/core/orchestrator/compiled_plate_execution.py b/openhcs/core/orchestrator/compiled_plate_execution.py index f2e49a305c..a5fe8a7921 100644 --- a/openhcs/core/orchestrator/compiled_plate_execution.py +++ b/openhcs/core/orchestrator/compiled_plate_execution.py @@ -104,6 +104,9 @@ class CompiledPlateExecutionExtras: RESULTS_SUMMARY_KEY: ClassVar[str] = "viewer_states_by_port" viewer_states_by_port: Mapping[int, "ViewerControlResponse"] + runtime_observation: RuntimeExecutionObservation = field( + default_factory=RuntimeExecutionObservation + ) class CompiledPlateExecutionResults(dict[str, ExecutionResult]): @@ -120,6 +123,13 @@ def __init__( viewer_states_by_port=MappingProxyType({}) ) + @property + def runtime_observations(self) -> tuple[RuntimeExecutionObservation, ...]: + return ( + *tuple(result.runtime_observation for result in self.values()), + self.extras.runtime_observation, + ) + @dataclass(frozen=True, slots=True) class CompiledPlateExecutionRequest(ProgressExecutionContext): @@ -249,6 +259,7 @@ def execute_compiled_plate_request( executor_resources.clear_execution_bundle() executor_resources.release_parent_runtime_resources(execution_bundle) + plate_runtime_observation = RuntimeExecutionObservation() if all(result.is_success() for result in execution_results.values()): plate_runtime_observation = execute_plate_scoped_steps( validated.compiled_contexts, @@ -285,7 +296,8 @@ def execute_compiled_plate_request( return CompiledPlateExecutionResults( execution_results, extras=CompiledPlateExecutionExtras( - viewer_states_by_port=viewer_states_by_port + viewer_states_by_port=viewer_states_by_port, + runtime_observation=plate_runtime_observation, ), ) except ExecutionCancelledError: @@ -445,6 +457,7 @@ def execute_plate_scoped_steps( plate_step_indexes = validate_plate_scoped_contexts(compiled_contexts) total_steps = max(next(iter(compiled_contexts.values())).step_plans) + 1 records_by_axis = _runtime_record_snapshot(compiled_contexts) + runtime_export_paths_by_context = {key: [] for key in compiled_contexts} for step_index in plate_step_indexes: owner_context, owner_plan = _plate_output_owner( compiled_contexts, @@ -523,13 +536,24 @@ def execute_plate_scoped_steps( ) records_by_axis = _records_with_output(records_by_axis, record) - RuntimeArtifactMaterializationAuthority.materialize( + materializations = RuntimeArtifactMaterializationAuthority.materialize( owner_context, owner_plan.require_function_execution_ready(), ) OpenHCSMetadataWriter.write( owner_context, owner_plan.require_function_execution_ready(), + artifact_materializations=materializations, + ) + owner_context_key = next( + key + for key, context in compiled_contexts.items() + if context is owner_context + ) + runtime_export_paths_by_context[owner_context_key].extend( + path + for materialization in materializations + for path in materialization.observation(owner_plan).runtime_export_paths ) _emit_execution_progress( progress_queue=progress_queue, @@ -547,6 +571,9 @@ def execute_plate_scoped_steps( RuntimeContextObservation( context_key=context_key, records=records, + runtime_export_paths=tuple( + dict.fromkeys(runtime_export_paths_by_context[context_key]) + ), ) for context_key, context in compiled_contexts.items() if ( diff --git a/openhcs/core/orchestrator/worker_execution.py b/openhcs/core/orchestrator/worker_execution.py index 7f82c7bafc..fe7776e554 100644 --- a/openhcs/core/orchestrator/worker_execution.py +++ b/openhcs/core/orchestrator/worker_execution.py @@ -48,11 +48,15 @@ from openhcs.core.progress.runtime_artifacts import ( runtime_artifact_context_for_records, ) -from openhcs.core.runtime_stores import StoredRuntimeValue +from openhcs.core.runtime_stores import ( + StoredRuntimeValue, + RuntimeArtifactAddress, + RuntimeArtifactLocation, +) from openhcs.core.steps.abstract import AbstractStep, StepExecutionObservation from openhcs.core.steps.function_artifact_materialization import ( - observed_materialized_artifact_locations_by_address, - observed_runtime_export_artifact_output_paths, + preview_reused_materialized_artifact_locations, + preview_reused_runtime_export_paths, ) from openhcs.utils.environment import OpenHCSProcessEnvironment @@ -63,21 +67,16 @@ def _runtime_observation_progress_context( records: tuple[StoredRuntimeValue, ...], *, - plan: CompiledStepPlan, - context: ProcessingContext, + materialized_locations_by_address: Mapping[ + RuntimeArtifactAddress, tuple[RuntimeArtifactLocation, ...] + ], ) -> dict | None: """Project one RuntimeValueStore observation delta through owned payloads.""" runtime_artifacts = runtime_artifact_context_for_records(records) live_measurements = live_measurement_context_for_records( records, - materialized_locations_by_address=( - observed_materialized_artifact_locations_by_address( - plan, - context, - records, - ) - ), + materialized_locations_by_address=materialized_locations_by_address, ) if runtime_artifacts is None: return live_measurements @@ -795,22 +794,16 @@ def _execute_axis_with_sequential_combinations( pipeline_definition, frozen_context, lane_context, + context_key=context_key, cancellation=cancellation, ) observed_records = runtime_store.observed_values_after( execution_observation_cursor ) runtime_export_paths = tuple( - dict.fromkeys( - path - for step_plan in frozen_context.step_plans.values() - if step_plan.owns_runtime_outputs - for path in observed_runtime_export_artifact_output_paths( - step_plan, - frozen_context, - observed_records, - ) - ) + path + for observation in result.runtime_observation.contexts + for path in observation.runtime_export_paths ) finally: # This cache is context-local even when lanes share a process. @@ -932,6 +925,8 @@ def _execute_single_axis_static( pipeline_definition: List[AbstractStep], frozen_context: ProcessingContext, lane_context: WorkerLaneExecutionContext, + *, + context_key: str, cancellation: ExecutionCancellationSignal | None = None, ) -> ExecutionResult: """Execute one frozen axis context against the compiled pipeline.""" @@ -952,6 +947,7 @@ def _execute_single_axis_static( frozen_context.bind_execution_runtime(lane_context) lane_context.install_debug_sink(frozen_context) runtime_value_store = frozen_context.runtime_value_store + runtime_export_paths = [] for step_index, step in enumerate(pipeline_definition): if cancellation is not None: @@ -981,8 +977,15 @@ def _execute_single_axis_static( ) runtime_progress_context = _runtime_observation_progress_context( observed_records, - plan=step_plan, - context=frozen_context, + # Historical debug previews are not a new-save observation. + materialized_locations_by_address=preview_reused_materialized_artifact_locations( + step_plan, frozen_context, observed_records + ), + ) + runtime_export_paths.extend( + preview_reused_runtime_export_paths( + step_plan, frozen_context, observed_records + ) ) emit( execution_id=lane_context.execution_id, @@ -1016,12 +1019,12 @@ def _execute_single_axis_static( ) observation_cursor = runtime_value_store.observation_cursor() - step.process(frozen_context, step_index) + step_observation = step.process(frozen_context, step_index) + runtime_export_paths.extend(step_observation.runtime_export_paths) observed_records = runtime_value_store.observed_values_after(observation_cursor) runtime_progress_context = _runtime_observation_progress_context( observed_records, - plan=step_plan, - context=frozen_context, + materialized_locations_by_address=step_observation.materialized_locations_by_address, ) emit( @@ -1044,7 +1047,18 @@ def _execute_single_axis_static( ): break - return ExecutionResult.success(axis_id=axis_id) + return ExecutionResult.success( + axis_id=axis_id, + runtime_observation=RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + context_key=context_key, + records=(), + runtime_export_paths=tuple(dict.fromkeys(runtime_export_paths)), + ), + ) + ), + ) def execute_worker_lane( diff --git a/openhcs/core/runtime_execution_validation.py b/openhcs/core/runtime_execution_validation.py index 8f1c90d4a1..b47296cc06 100644 --- a/openhcs/core/runtime_execution_validation.py +++ b/openhcs/core/runtime_execution_validation.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections import Counter -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from types import MappingProxyType @@ -12,6 +12,7 @@ from openhcs.core.artifacts import ArtifactSpec, ArtifactType from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.orchestrator.execution_result import RuntimeExecutionObservation from openhcs.core.runtime_exports import ( RuntimeExportExpectation, RuntimeExportObservation, @@ -235,12 +236,13 @@ def runtime_artifact_viewer_expectations( if not plan.streaming_configs or not plan.owns_runtime_outputs: continue from openhcs.core.steps.function_artifact_materialization import ( - observed_runtime_artifact_materializations, + runtime_artifact_materializations_from_records, ) - for materialization in observed_runtime_artifact_materializations( + for materialization in runtime_artifact_materializations_from_records( plan, context, + context.runtime_value_store.observed_values, ): output_plan = materialization.output_plan if plan.compiled_function_pattern.publishes_output_to_main_flow( @@ -319,6 +321,8 @@ class RuntimeArtifactExecutionObservation: def from_contexts( cls, execution_contexts: Mapping[str, ProcessingContext], + *, + runtime_observations: Sequence[RuntimeExecutionObservation], ) -> "RuntimeArtifactExecutionObservation": identity_policies = frozenset( context.source_image_set_identity_policy @@ -331,8 +335,8 @@ def from_contexts( ) return cls( records_by_axis=runtime_records_by_axis(execution_contexts), - exports=RuntimeExportObservation.from_execution_contexts( - execution_contexts + exports=RuntimeExportObservation.from_runtime_observations( + runtime_observations ), source_image_set_identity_policy=next( iter(identity_policies), diff --git a/openhcs/core/runtime_exports.py b/openhcs/core/runtime_exports.py index 1cc9d50317..ee5f64cc05 100644 --- a/openhcs/core/runtime_exports.py +++ b/openhcs/core/runtime_exports.py @@ -28,7 +28,6 @@ from openhcs.processing.materialization.core import materialization_is_empty if TYPE_CHECKING: - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.orchestrator.execution_result import RuntimeExecutionObservation @@ -113,26 +112,6 @@ class RuntimeExportObservation: table_row_counts_by_path: Mapping[Path, int] output_files: tuple[Path, ...] = () - @classmethod - def from_execution_contexts( - cls, - execution_contexts: Mapping[str, ProcessingContext], - ) -> RuntimeExportObservation: - """Read contract-owned export paths without retaining runtime values.""" - from openhcs.core.steps.function_artifact_materialization import ( - runtime_export_artifact_output_paths, - ) - - return cls.from_output_paths( - tuple( - path - for context in execution_contexts.values() - for plan in context.step_plans.values() - if plan.owns_runtime_outputs - for path in runtime_export_artifact_output_paths(plan, context) - ) - ) - @classmethod def from_runtime_observations( cls, diff --git a/openhcs/core/steps/abstract.py b/openhcs/core/steps/abstract.py index e82003ec93..19402fa4d3 100644 --- a/openhcs/core/steps/abstract.py +++ b/openhcs/core/steps/abstract.py @@ -8,10 +8,11 @@ import abc import inspect +from pathlib import Path from collections.abc import Mapping -from dataclasses import dataclass, is_dataclass +from dataclasses import dataclass, field, is_dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, get_type_hints +from typing import TYPE_CHECKING, Iterable, get_type_hints from objectstate import get_base_type_for_lazy, semantic_values_equal @@ -49,10 +50,29 @@ class StepExecutionObservation: tuple[RuntimeArtifactLocation, ...], ] + runtime_export_paths: tuple[Path, ...] = field(default_factory=tuple) + @classmethod def empty(cls) -> "StepExecutionObservation": return cls(MappingProxyType({})) + @classmethod + def combine( + cls, observations: Iterable["StepExecutionObservation"] + ) -> "StepExecutionObservation": + locations = {} + paths = [] + for observation in observations: + for ( + address, + values, + ) in observation.materialized_locations_by_address.items(): + locations[address] = tuple( + dict.fromkeys((*locations.get(address, ()), *values)) + ) + paths.extend(observation.runtime_export_paths) + return cls(MappingProxyType(locations), tuple(dict.fromkeys(paths))) + # def get_step_id(step: 'AbstractStep') -> str: # """ diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index 7d4a0da320..7510f0f61d 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, cast import numpy as np @@ -50,6 +51,7 @@ with_source_component_metadata, ) from openhcs.core.source_projection import OpenHCSPlaneAddress +from openhcs.core.steps.abstract import StepExecutionObservation from openhcs.core.steps.function_output_identity import ( FunctionOutputIdentity, FunctionOutputIdentityAuthority, @@ -71,9 +73,11 @@ MaterializationSpec, MaterializationValue, Output, + SavedMaterializationOutputs, RawBackendKwargs, ViewerStreamBackendCallKwargs, materialization_outputs, + prepare_materialization, ) if TYPE_CHECKING: @@ -1221,18 +1225,6 @@ def runtime_artifact_materializations( return tuple(materializations) -def observed_runtime_artifact_materializations( - plan: CompiledStepPlan, - context: "ProcessingContext", -) -> tuple[RuntimeArtifactMaterialization, ...]: - """Derive historical materializations from the runtime observation ledger.""" - return runtime_artifact_materializations_from_records( - plan, - context, - context.runtime_value_store.observed_values, - ) - - def runtime_artifact_materializations_from_records( plan: CompiledStepPlan, context: "ProcessingContext", @@ -1271,62 +1263,12 @@ def runtime_artifact_materializations_from_records( return tuple(materializations) -def materialized_artifact_output_paths( - plan: CompiledStepPlan, - context: "ProcessingContext", -) -> tuple[Path, ...]: - """Re-derive exact persistent output paths from runtime materializations.""" - - if not plan.runtime_artifact_materialization.has_persistent_target: - return () - return tuple( - Path(output.path) - for materialization in runtime_artifact_materializations(plan, context) - if materialization.spec.participates_in_persistent_materialization() - for output in materialization.outputs(plan, context) - ) - - -def runtime_export_artifact_output_paths( - plan: CompiledStepPlan, - context: "ProcessingContext", -) -> tuple[Path, ...]: - """Derive exact pipeline-declared export paths from runtime materializations.""" - - if not plan.runtime_artifact_materialization.has_persistent_target: - return () - return tuple( - Path(output.path) - for materialization in runtime_artifact_materializations(plan, context) - if materialization.spec.participates_in_runtime_export_observation() - for output in materialization.outputs(plan, context) - ) - - -def observed_materialized_artifact_output_paths( +def preview_reused_runtime_export_paths( plan: CompiledStepPlan, context: "ProcessingContext", records: tuple[StoredRuntimeValue, ...], ) -> tuple[Path, ...]: - """Derive exact persistent outputs from one execution-owned observation.""" - - return tuple( - Path(location.path) - for locations in observed_materialized_artifact_locations_by_address( - plan, - context, - records, - ).values() - for location in locations - ) - - -def observed_runtime_export_artifact_output_paths( - plan: CompiledStepPlan, - context: "ProcessingContext", - records: tuple[StoredRuntimeValue, ...], -) -> tuple[Path, ...]: - """Derive exact pipeline-declared exports from worker-observed records.""" + """Predict historical debug-reuse exports, independently of new save outcomes.""" if not plan.runtime_artifact_materialization.has_persistent_target: return () @@ -1342,12 +1284,12 @@ def observed_runtime_export_artifact_output_paths( ) -def observed_materialized_artifact_locations_by_address( +def preview_reused_materialized_artifact_locations( plan: CompiledStepPlan, context: "ProcessingContext", records: tuple[StoredRuntimeValue, ...], ) -> Mapping[RuntimeArtifactAddress, tuple[RuntimeArtifactLocation, ...]]: - """Project exact persistent destinations for observed runtime artifacts.""" + """Predict historical debug-reuse destinations from retained logical values.""" if not plan.runtime_artifact_materialization.has_persistent_target: return {} @@ -1432,15 +1374,41 @@ def _planned_materialization_path( ) +@dataclass(frozen=True, slots=True) +class MaterializedRuntimeArtifact(SavedMaterializationOutputs): + """Actual writer outputs saved for one reduced runtime artifact.""" + + materialization: RuntimeArtifactMaterialization + + def observation(self, plan: CompiledStepPlan) -> StepExecutionObservation: + target = plan.runtime_artifact_materialization + if not target.has_persistent_target: + return StepExecutionObservation.empty() + backend = target.require_persistent_backend() + outputs = self.outputs_for_backend(backend) + if not outputs: + return StepExecutionObservation.empty() + address = RuntimeArtifactAddress.from_record(self.materialization.record) + locations = tuple( + RuntimeArtifactLocation(path=output.path, backend=backend) + for output in outputs + ) + paths = ( + tuple(Path(output.path) for output in outputs) + if self.materialization.spec.participates_in_runtime_export_observation() + else () + ) + return StepExecutionObservation(MappingProxyType({address: locations}), paths) + + def materialize_artifact_outputs( filemanager: "FileManager", plan: CompiledStepPlan, target_plan: ArtifactMaterializationTargetPlan, context: "ProcessingContext", -) -> None: - """Materialize planned artifact outputs to persistent and streaming backends.""" - from openhcs.processing.materialization import materialize - +) -> tuple[MaterializedRuntimeArtifact, ...]: + """Save each exact artifact batch once and return its successful outputs.""" + saved_materializations = [] images_dir = plan.artifact_images_dir for materialization in runtime_artifact_materializations(plan, context): @@ -1458,7 +1426,7 @@ def materialize_artifact_outputs( ) if not backends: continue - materialize( + batch = prepare_materialization( materialization.spec, data, str(materialization.base_path), @@ -1488,3 +1456,12 @@ def materialize_artifact_outputs( pipeline_position=plan.pipeline_position, output_plan=materialization.output_plan, ) + saved_materializations.append( + MaterializedRuntimeArtifact( + outputs_by_backend=MappingProxyType( + dict(batch.save().outputs_by_backend) + ), + materialization=materialization, + ) + ) + return tuple(saved_materializations) diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py index 919d4acaa5..eda426ccc8 100644 --- a/openhcs/core/steps/function_outputs.py +++ b/openhcs/core/steps/function_outputs.py @@ -6,9 +6,9 @@ import time from abc import ABC, abstractmethod from collections.abc import Callable, Mapping -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from pathlib import Path -from typing import ClassVar +from typing import ClassVar, TypeVar import numpy as np from metaclass_registry import ( @@ -55,11 +55,12 @@ SourceProjectionMetadataSerializer, SourceProjectionSet, ) +from openhcs.core.steps.abstract import StepExecutionObservation from openhcs.core.steps.function_artifact_materialization import ( + MaterializedRuntimeArtifact, PersistentArtifactMaterializationTargetPlan, StreamingOnlyArtifactMaterializationTargetPlan, materialize_artifact_outputs, - runtime_artifact_materializations, ) from openhcs.core.steps.function_io import ( prepare_storage_image_payloads, @@ -137,50 +138,61 @@ def memory_path( def finalize_function_step_outputs( context: ProcessingContext, plan: CompiledStepPlan, -) -> None: - """Persist images, streams, metadata, and non-image artifacts for one step.""" +) -> StepExecutionObservation: + """Save and publish one step's actual outputs before releasing their payloads.""" if not RuntimeProfileLogger.enabled(): MemoryOutputWriter.write_if_needed(context, plan) MaterializedImageOutputWriter.write_if_needed(context, plan) StreamOutputsAuthority.stream_outputs(context, plan) - RuntimeArtifactMaterializationAuthority.materialize(context, plan) - OpenHCSMetadataWriter.write(context, plan) - return - - _profile_finalization_phase( - "finalize_memory_outputs", - lambda: MemoryOutputWriter.write_if_needed(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_materialized_images", - lambda: MaterializedImageOutputWriter.write_if_needed(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_stream_outputs", - lambda: StreamOutputsAuthority.stream_outputs(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_runtime_artifacts", - lambda: RuntimeArtifactMaterializationAuthority.materialize(context, plan), - plan, - ) - _profile_finalization_phase( - "finalize_openhcs_metadata", - lambda: OpenHCSMetadataWriter.write(context, plan), - plan, + materializations = RuntimeArtifactMaterializationAuthority.materialize( + context, plan + ) + OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ) + else: + _profile_finalization_phase( + "finalize_memory_outputs", + lambda: MemoryOutputWriter.write_if_needed(context, plan), + plan, + ) + _profile_finalization_phase( + "finalize_materialized_images", + lambda: MaterializedImageOutputWriter.write_if_needed(context, plan), + plan, + ) + _profile_finalization_phase( + "finalize_stream_outputs", + lambda: StreamOutputsAuthority.stream_outputs(context, plan), + plan, + ) + materializations = _profile_finalization_phase( + "finalize_runtime_artifacts", + lambda: RuntimeArtifactMaterializationAuthority.materialize(context, plan), + plan, + ) + _profile_finalization_phase( + "finalize_openhcs_metadata", + lambda: OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ), + plan, + ) + return StepExecutionObservation.combine( + item.observation(plan) for item in materializations ) +_FinalizationResult = TypeVar("_FinalizationResult") + + def _profile_finalization_phase( label: str, - operation: Callable[[], None], + operation: Callable[[], _FinalizationResult], plan: CompiledStepPlan, -) -> None: +) -> _FinalizationResult: started_at = time.perf_counter() - operation() + result = operation() RuntimeProfileLogger.log( logger, label, @@ -189,6 +201,7 @@ def _profile_finalization_phase( step_name=plan.step_name, axis_id=plan.axis_id, ) + return result class MemoryOutputWriter: @@ -691,6 +704,9 @@ class OutputTarget(ABC, metaclass=AutoRegisterMeta): plate_root: str sub_dir: str results_dir: str | None + artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = field( + default=(), compare=False, hash=False, repr=False + ) @classmethod @abstractmethod @@ -719,17 +735,25 @@ def from_execution( @classmethod def for_execution( - cls, context: ProcessingContext, plan: CompiledStepPlan + cls, + context: ProcessingContext, + plan: CompiledStepPlan, + *, + artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = (), ) -> tuple[OpenHCSMetadataWriter.OutputTarget, ...]: return tuple( target for declaration in cls.__registry__.values() if (owner := declaration.from_execution(context, plan)) is not None - for target in owner.production_targets(context, plan) + for target in replace( + owner, artifact_materializations=artifact_materializations + ).production_targets(context, plan) ) def production_targets( - self, context: ProcessingContext, plan: CompiledStepPlan + self, + context: ProcessingContext, + plan: CompiledStepPlan, ) -> tuple[OpenHCSMetadataWriter.OutputTarget, ...]: """Project this declaration's exact storage destinations for the step.""" return (self,) @@ -747,7 +771,9 @@ def produced_records( return () def runtime_artifact_projection_paths( - self, context: ProcessingContext, plan: CompiledStepPlan + self, + context: ProcessingContext, + plan: CompiledStepPlan, ) -> tuple[tuple[SourceArtifactProjection, str], ...]: """Publish artifacts persisted in this declared storage target.""" materialization = plan.runtime_artifact_materialization @@ -820,7 +846,9 @@ def produced_projection_metadata( if context.filemanager is None: raise ValueError("OpenHCS metadata requires a file manager.") target = type(self).from_plan(plan) - if target is None or self not in target.production_targets(context, plan): + if target is None or self not in replace( + target, artifact_materializations=self.artifact_materializations + ).production_targets(context, plan): raise ValueError( "Produced metadata plan does not own this output target." ) @@ -910,16 +938,9 @@ def project_runtime_artifacts( """Project persisted image artifacts into the target source authority.""" projection_paths = [] - for materialization in runtime_artifact_materializations(plan, context): - if ( - not materialization.spec.participates_in_persistent_materialization() - ): - continue - for output in materialization.outputs( - plan, - context, - output_path_filter=ImageFileFormat.is_image_path, - ): + for saved_artifact in self.artifact_materializations: + materialization = saved_artifact.materialization + for output in saved_artifact.outputs_for_backend(self.backend): if not ImageFileFormat.is_image_path(output.path) or Path( output.path ).parent != Path(self.output_dir): @@ -1005,8 +1026,12 @@ def write( cls, context: ProcessingContext, plan: CompiledStepPlan, + *, + artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = (), ) -> None: - for target in cls.OutputTarget.for_execution(context, plan): + for target in cls.OutputTarget.for_execution( + context, plan, artifact_materializations=artifact_materializations + ): if not plan.create_openhcs_metadata: structured_metadata = target.produced_projection_metadata(context, plan) if structured_metadata is None: @@ -1124,23 +1149,21 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): """Image artifacts persist independently of main-flow image/checkpoint storage.""" def production_targets( - self, context: ProcessingContext, plan: CompiledStepPlan + self, + context: ProcessingContext, + plan: CompiledStepPlan, ) -> tuple[RuntimeArtifactMetadataTarget, ...]: """Derive directories from the same writer outputs used to save artifacts.""" directories = dict.fromkeys( Path(output.path).parent - for materialization in runtime_artifact_materializations(plan, context) - if materialization.spec.participates_in_persistent_materialization() - for output in materialization.outputs( - plan, context, output_path_filter=ImageFileFormat.is_image_path - ) + for materialization in self.artifact_materializations + for output in materialization.outputs_for_backend(self.backend) if ImageFileFormat.is_image_path(output.path) ) return tuple( target for directory in directories for target in (self.for_directory(directory),) - if target.contains_images(context) ) def reconciliation_targets( @@ -1193,27 +1216,28 @@ def materialize( cls, context: ProcessingContext, plan: CompiledStepPlan, - ) -> None: + ) -> tuple[MaterializedRuntimeArtifact, ...]: if not plan.artifact_outputs: - return + return () materialization_plan = plan.runtime_artifact_materialization has_persistent_target = materialization_plan.has_persistent_target has_streaming_target = bool(plan.streaming_configs) if not has_persistent_target and not has_streaming_target: logger.info("Skipping runtime artifact materialization and streaming") - return + return () logger.info( "Starting materialization for %s artifact outputs", len(plan.artifact_outputs), ) - materialize_artifact_outputs( + materializations = materialize_artifact_outputs( context.filemanager, plan, cls.target_plan(materialization_plan), context, ) logger.info("Completed artifact materialization") + return materializations @staticmethod def target_plan( diff --git a/openhcs/processing/materialization/__init__.py b/openhcs/processing/materialization/__init__.py index 4e920f0ee9..c161998f13 100644 --- a/openhcs/processing/materialization/__init__.py +++ b/openhcs/processing/materialization/__init__.py @@ -6,11 +6,14 @@ ) from openhcs.processing.materialization.core import ( BackendSaver, + MaterializationBatch, MaterializationContext, MaterializationSpec, Output, + SavedMaterializationOutputs, PathHelper, materialize, + prepare_materialization, materialization_outputs, registered_materialization_option_types, tabular_field_names_from_materialization, @@ -57,11 +60,14 @@ "MaterializationSpec", "TerminalMaterializationSpec", "StreamingOnlyMaterializationSpec", + "MaterializationBatch", "MaterializationContext", "Output", + "SavedMaterializationOutputs", "PathHelper", "BackendSaver", "materialize", + "prepare_materialization", "materialization_outputs", "registered_materialization_option_types", "FileOutputOptions", diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index faff29684a..a095267a20 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -17,6 +17,7 @@ from dataclasses import dataclass, field, is_dataclass, replace from functools import lru_cache, singledispatch from pathlib import Path, PurePosixPath, PureWindowsPath +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, TypeAlias import numpy as np @@ -1651,7 +1652,8 @@ def __init__( def save_all( self, outputs: Sequence[Output], - ) -> None: + ) -> Mapping[str, tuple[Output, ...]]: + saved_outputs: dict[str, tuple[Output, ...]] = {} for backend in self.backends: backend_instance = self.filemanager._get_backend(backend) output_acceptance = tuple( @@ -1690,6 +1692,8 @@ def save_all( backend, **kwargs, ) + saved_outputs[backend] = supported_outputs + return saved_outputs def _prepare_path(self, backend: str, backend_instance, path: str) -> None: if not backend_instance.requires_filesystem_validation: @@ -4129,28 +4133,69 @@ def validate(spec: MaterializationSpec, backends: list[str]) -> None: ) -def _materialization_output_groups( - spec: MaterializationSpec, - data: MaterializationValue, - context: MaterializationContext, - *, - output_path_filter: Callable[[Path], bool] | None = None, -) -> tuple[tuple[WriterSpec, tuple[Output, ...]], ...]: - """Render requested outputs through each writer's declared path projection.""" +@dataclass(frozen=True) +class SavedMaterializationOutputs: + """Backend-indexed outputs from completed saves, independent of artifact kind.""" - return tuple( - ( - writer, - tuple( - output.with_source_identity_fallback( - context.artifact_source_identity - ).with_variable_components(context.variable_components) - for output in writer.outputs(data, options, context, output_path_filter) - ), + outputs_by_backend: Mapping[str, tuple[Output, ...]] + + def outputs_for_backend(self, backend: str) -> tuple[Output, ...]: + return self.outputs_by_backend.get(backend, ()) + + +@dataclass(frozen=True) +class MaterializationBatch: + """One rendered writer batch shared by saving and immediate publication.""" + + context: MaterializationContext + output_groups: tuple[tuple[WriterSpec, tuple[Output, ...]], ...] + primary: int + + @classmethod + def render( + cls, + spec: MaterializationSpec, + data: MaterializationValue, + context: MaterializationContext, + *, + output_path_filter: Callable[[Path], bool] | None = None, + ) -> MaterializationBatch: + groups = tuple( + ( + writer, + tuple( + output.with_source_identity_fallback( + context.artifact_source_identity + ).with_variable_components(context.variable_components) + for output in writer.outputs( + data, options, context, output_path_filter + ) + ), + ) + for options in spec.outputs + for writer in (_WRITERS_BY_OPTIONS[options.__class__],) + ) + return cls(context=context, output_groups=groups, primary=spec.primary) + + @property + def outputs(self) -> tuple[Output, ...]: + return tuple( + output for _writer, outputs in self.output_groups for output in outputs ) - for options in spec.outputs - for writer in (_WRITERS_BY_OPTIONS[options.__class__],) - ) + + @property + def primary_path(self) -> str: + writer, outputs = self.output_groups[self.primary] + return writer.primary_path(list(outputs)) + + def save(self) -> SavedMaterializationOutputs: + """Return only outputs accepted and successfully saved by each backend.""" + saved: dict[str, tuple[Output, ...]] = {} + saver = self.context.saver + for _writer, outputs in self.output_groups: + for backend, saved_outputs in saver.save_all(outputs).items(): + saved[backend] = (*saved.get(backend, ()), *saved_outputs) + return SavedMaterializationOutputs(MappingProxyType(saved)) def materialization_outputs( @@ -4190,19 +4235,15 @@ def materialization_outputs( pipeline_position=pipeline_position, output_plan=output_plan, ) - return tuple( - output - for _writer, outputs in _materialization_output_groups( - spec, - data, - materialization_context, - output_path_filter=output_path_filter, - ) - for output in outputs - ) + return MaterializationBatch.render( + spec, + data, + materialization_context, + output_path_filter=output_path_filter, + ).outputs -def materialize( +def prepare_materialization( spec: MaterializationSpec, data: MaterializationValue, path: str, @@ -4218,8 +4259,8 @@ def materialize( source_paths: Sequence[str] = (), pipeline_position: int | None = None, output_plan: ArtifactOutputPlan | None = None, -) -> str: - """Materialize data to one or more backends.""" +) -> MaterializationBatch: + """Prepare one exact batch for saving and immediate publication.""" normalized_backends = BackendSequenceAuthority.normalize(backends) AllowedBackendsAuthority.validate(spec, normalized_backends) @@ -4245,11 +4286,42 @@ def materialize( output_plan=output_plan, ) - primary_path = "" + return MaterializationBatch.render(spec, data, ctx) - for i, (writer, outs) in enumerate(_materialization_output_groups(spec, data, ctx)): - ctx.saver.save_all(outs) - if i == spec.primary: - primary_path = writer.primary_path(list(outs)) - return primary_path +def materialize( + spec: MaterializationSpec, + data: MaterializationValue, + path: str, + filemanager: FileManager, + backends: Sequence[str] | str, + backend_kwargs: BackendKwargsInput = BACKEND_KWARGS_ABSENT, + context: ProcessingContext | None = None, + extra_inputs: dict | None = None, + *, + artifact_source_identity: SourceImageIdentity | None = None, + artifact_filename_identity: SourceImageIdentity | None = None, + variable_components: Sequence[VariableComponents] = (), + source_paths: Sequence[str] = (), + pipeline_position: int | None = None, + output_plan: ArtifactOutputPlan | None = None, +) -> str: + """Materialize data and return the primary path derived from its writer batch.""" + batch = prepare_materialization( + spec, + data, + path, + filemanager, + backends, + backend_kwargs, + context=context, + extra_inputs=extra_inputs, + artifact_source_identity=artifact_source_identity, + artifact_filename_identity=artifact_filename_identity, + variable_components=variable_components, + source_paths=source_paths, + pipeline_position=pipeline_position, + output_plan=output_plan, + ) + batch.save() + return batch.primary_path diff --git a/openhcs/runtime/zmq_execution_observation.py b/openhcs/runtime/zmq_execution_observation.py index fecffce412..f75dc3072d 100644 --- a/openhcs/runtime/zmq_execution_observation.py +++ b/openhcs/runtime/zmq_execution_observation.py @@ -9,7 +9,11 @@ from pathlib import Path from openhcs.core.context.processing_context import ProcessingContext -from openhcs.core.orchestrator.execution_result import ExecutionResult, ExecutionStatus +from openhcs.core.orchestrator.execution_result import ( + ExecutionResult, + RuntimeExecutionObservation, + ExecutionStatus, +) from openhcs.core.runtime_execution_validation import ( RuntimeArtifactExecutionExpectation, RuntimeArtifactExecutionObservation, @@ -215,11 +219,12 @@ def from_execution( compiled_contexts: Mapping[str, ProcessingContext], execution_results: Mapping[str, ExecutionResult], output_roots: tuple[Path, ...], + runtime_observations: tuple[RuntimeExecutionObservation, ...], server_environment: RuntimeEnvironmentSnapshot | None = None, execution_id: str | None = None, ) -> ZMQRuntimeExecutionObservationExport: observation = RuntimeArtifactExecutionObservation.from_contexts( - compiled_contexts + compiled_contexts, runtime_observations=runtime_observations ) return cls( schema_version=ZMQ_RUNTIME_OBSERVATION_EXPORT_SCHEMA_VERSION, diff --git a/openhcs/runtime/zmq_execution_server.py b/openhcs/runtime/zmq_execution_server.py index ce858a0a2f..0c2bcce4d5 100644 --- a/openhcs/runtime/zmq_execution_server.py +++ b/openhcs/runtime/zmq_execution_server.py @@ -770,17 +770,8 @@ def _export_runtime_observation( is ZMQRuntimeObservationExportScope.OUTCOMES ): observed_exports = RuntimeExportObservation.from_runtime_observations( - tuple( - result.runtime_observation for result in execution_results.values() - ) + execution_results.runtime_observations ) - if execution_bundle.requires_parent_runtime_observation: - parent_exports = RuntimeExportObservation.from_execution_contexts( - execution_bundle.runtime_contexts - ) - observed_exports = RuntimeExportObservation.from_output_paths( - (*observed_exports.output_files, *parent_exports.output_files) - ) export = ZMQRuntimeExecutionOutcomeExport.from_execution( compiled_axis_ids=execution_bundle.runtime_contexts, execution_results=execution_results, @@ -793,6 +784,7 @@ def _export_runtime_observation( export = ZMQRuntimeExecutionObservationExport.from_execution( compiled_contexts=execution_bundle.runtime_contexts, execution_results=execution_results, + runtime_observations=execution_results.runtime_observations, output_roots=output_roots, server_environment=self._server_environment, execution_id=request_context.execution_id, diff --git a/tests/unit/test_function_artifact_materialization.py b/tests/unit/test_function_artifact_materialization.py index bca6321c5a..2a889182ae 100644 --- a/tests/unit/test_function_artifact_materialization.py +++ b/tests/unit/test_function_artifact_materialization.py @@ -119,13 +119,10 @@ StreamingOnlyArtifactMaterializationTargetPlan, actual_materialization_records, materialize_artifact_outputs, - materialized_artifact_output_paths, - observed_materialized_artifact_locations_by_address, - observed_materialized_artifact_output_paths, + preview_reused_materialized_artifact_locations, planned_materialization_preview, runtime_artifact_materializations, runtime_artifact_materializations_from_records, - runtime_export_artifact_output_paths, ) from openhcs.core.steps.function_output_identity import ( IncompleteFunctionOutputFilenameIdentityError, @@ -155,6 +152,7 @@ from openhcs.processing.materialization.core import ( MaterializationSpec, Output, + SavedMaterializationOutputs, materialization_outputs, ) from openhcs.processing.materialization.options import ( @@ -789,10 +787,10 @@ def test_materialize_artifact_outputs_uses_runtime_store_payload( def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -839,10 +837,10 @@ def test_materialize_artifact_outputs_attaches_image_schema_provenance(monkeypat def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -922,10 +920,10 @@ def fake_materialize(spec, data, path, manager, *_args, **kwargs): variable_components=kwargs["variable_components"], ) ) - return output_paths[0] + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -940,7 +938,14 @@ def fake_materialize(spec, data, path, manager, *_args, **kwargs): expected_paths = tuple( Path(f"/images/A01_s{site:03d}_w1_z001_t001_saved.tif") for site in range(1, 4) ) - assert materialized_artifact_output_paths(plan, context) == expected_paths + assert ( + tuple( + Path(output.path) + for materialization in runtime_artifact_materializations(plan, context) + for output in materialization.outputs(plan, context) + ) + == expected_paths + ) materialize_artifact_outputs( filemanager, @@ -990,10 +995,10 @@ def test_materialize_artifact_outputs_does_not_require_vfs_payload_for_store_rec def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -1045,10 +1050,10 @@ def test_materialize_artifact_outputs_uses_runtime_record_identity_not_final_pat def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -1099,10 +1104,10 @@ def test_materialize_artifact_outputs_uses_declared_measurement_csv_spec( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -1711,17 +1716,19 @@ def record_for_site(site: str) -> None: current_execution_records = context.runtime_value_store.observed_values_after( current_execution_cursor ) - assert observed_materialized_artifact_output_paths( - plan, - context, - current_execution_records, + assert tuple( + Path(location.path) + for locations in preview_reused_materialized_artifact_locations( + plan, context, current_execution_records + ).values() + for location in locations ) == ( Path( "/analysis/A01_site-2_z_index-1_timepoint-1_" "cell_counts_step7_details.csv" ), ) - assert observed_materialized_artifact_locations_by_address( + assert preview_reused_materialized_artifact_locations( plan, context, current_execution_records, @@ -1794,7 +1801,7 @@ def test_terminal_persistence_is_reported_without_becoming_declared_export() -> persistent_backend="disk", ) - locations = observed_materialized_artifact_locations_by_address( + locations = preview_reused_materialized_artifact_locations( plan, context, (record,), @@ -1808,7 +1815,6 @@ def test_terminal_persistence_is_reported_without_becoming_declared_export() -> ), ), } - assert runtime_export_artifact_output_paths(plan, context) == () assert not output_plan.materialization.participates_in_runtime_export_observation() @@ -1929,10 +1935,10 @@ def test_materialize_artifact_outputs_unions_measurement_subject_records( def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((tuple(data.iter_row_mappings()), path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2232,10 +2238,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2298,10 +2304,10 @@ def test_materialize_artifact_outputs_uses_actual_group_records(monkeypatch): def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2578,10 +2584,10 @@ def test_materialize_artifact_outputs_uses_group_measurement_artifact_identity( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2642,10 +2648,10 @@ def test_materialize_artifact_outputs_keeps_grouped_artifact_record_path( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2708,10 +2714,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2778,10 +2784,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2928,10 +2934,10 @@ def test_materialize_artifact_outputs_uses_declared_metadata_json_spec( def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -2962,9 +2968,10 @@ def test_materialize_artifact_outputs_skips_special_without_explicit_spec( def fake_materialize(*args, **kwargs): materialized.append((args, kwargs)) + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3057,9 +3064,10 @@ def test_materialize_artifact_outputs_skips_explicitly_disabled_artifact_without def fake_materialize(*args, **kwargs): materialized.append((args, kwargs)) + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3097,10 +3105,10 @@ def test_materialize_artifact_outputs_uses_declared_object_labels_roi_spec(monke def fake_materialize(spec, data, path, *_args, **_kwargs): materialized.append((spec, data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3154,10 +3162,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3243,9 +3251,10 @@ def fake_materialize( **_kwargs, ): materialized.append((backends, backend_kwargs)) + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3307,10 +3316,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3389,10 +3398,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3451,10 +3460,10 @@ def test_materialize_artifact_outputs_uses_runtime_plane_group_identity( def fake_materialize(_spec, data, path, *_args, **_kwargs): materialized.append((data, path)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3522,10 +3531,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3623,10 +3632,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3795,10 +3804,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3898,10 +3907,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -3984,10 +3993,10 @@ def test_materialize_image_uses_declared_filename_source_identity(monkeypatch): def fake_materialize(_spec, _data, path, *_args, **_kwargs): materialized.append(path) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) @@ -4144,7 +4153,11 @@ def test_compiled_z_axis_reaches_source_named_image_materialization() -> None: assert image_payload_metadata(saved_payload).plane_axis is ( RuntimePlaneAxis.RUNTIME_SLICE ) - assert materialized_artifact_output_paths(plan, context) == ( + assert tuple( + Path(output.path) + for materialization in runtime_artifact_materializations(plan, context) + for output in materialization.outputs(plan, context) + ) == ( Path("/images/A01_s001_w2_z001_t001_NucleiLabels.tif"), Path("/images/A01_s001_w2_z002_t001_NucleiLabels.tif"), ) @@ -4214,10 +4227,10 @@ def fake_materialize( **_kwargs, ): materialized.append((spec, data, path, backends, backend_kwargs)) - return path + return SimpleNamespace(save=lambda: SavedMaterializationOutputs({})) monkeypatch.setattr( - "openhcs.processing.materialization.materialize", + "openhcs.core.steps.function_artifact_materialization.prepare_materialization", fake_materialize, ) diff --git a/tests/unit/test_function_outputs.py b/tests/unit/test_function_outputs.py index 3e2a76a8fc..a97b9bbb33 100644 --- a/tests/unit/test_function_outputs.py +++ b/tests/unit/test_function_outputs.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from dataclasses import replace from types import SimpleNamespace import numpy as np @@ -39,6 +40,7 @@ from openhcs.core.runtime_image_values import ( ImageMetadataPayload, ImagePayloadMetadata, + image_payload_metadata, ) from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.source_image_provenance import ( @@ -66,6 +68,10 @@ ProducedOutputSemantics, step_output_manifest, ) +from openhcs.core.steps.function_artifact_materialization import ( + MaterializedRuntimeArtifact, + RuntimeArtifactMaterialization, +) from openhcs.core.steps.function_outputs import ( MaterializedImageOutputWriter, MemoryOutputWriter, @@ -97,6 +103,7 @@ MaterializedFilenameIdentity, ) + @pytest.mark.parametrize("backend", [Backend.ZARR.value, "custom-array-store"]) def test_memory_output_writer_projects_runtime_image_payload_for_array_storage( backend, @@ -137,7 +144,9 @@ def test_function_step_metadata_follows_runtime_artifact_persistence( monkeypatch.setattr( authority, method_name, - lambda _context, _plan, event=event: events.append(event), + lambda _context, _plan, event=event, **_kwargs: (events.append(event), ())[ + 1 + ], ) finalize_function_step_outputs(context, plan) @@ -1371,10 +1380,13 @@ def test_image_persistence_skips_object_label_main_flow_payloads(): assert ProducedMemoryPathsAuthority.paths(context, plan) == [] -def test_metadata_target_family_discovers_new_declaration_without_consumer_edits(tmp_path): +def test_metadata_target_family_discovers_new_declaration_without_consumer_edits( + tmp_path, +): registry = OpenHCSMetadataWriter.OutputTarget.__registry__ original_keys = set(registry) try: + class SupplementalImageMetadataTarget(OpenHCSMetadataWriter.OutputTarget): @classmethod def from_plan(cls, plan): @@ -1391,7 +1403,9 @@ def from_plan(cls, plan): context = context_stub(FileManagerStub({})) (target,) = OpenHCSMetadataWriter.OutputTarget.for_plan(plan) assert type(target) is SupplementalImageMetadataTarget - assert OpenHCSMetadataWriter.OutputTarget.for_execution(context, plan) == (target,) + assert OpenHCSMetadataWriter.OutputTarget.for_execution(context, plan) == ( + target, + ) assert target.produced_projection_metadata(context, plan) is None finally: for key in set(registry) - original_keys: @@ -1638,13 +1652,16 @@ def test_runtime_image_artifact_projects_persisted_source_binding( ), outputs=lambda _plan, _context, **_kwargs: (output,), ) - monkeypatch.setattr( - "openhcs.core.steps.function_outputs.runtime_artifact_materializations", - lambda _plan, _context: (materialization,), + saved_artifacts = ( + MaterializedRuntimeArtifact( + outputs_by_backend={Backend.DISK.value: (output,)}, + materialization=materialization, + ), ) target = MaterializedImageMetadataTarget.from_plan(plan) assert target is not None + target = replace(target, artifact_materializations=saved_artifacts) [(projection, virtual_path)] = target.runtime_artifact_projection_paths( context, plan ) @@ -1755,9 +1772,11 @@ def test_runtime_multiplane_label_artifact_projects_persisted_source_binding( ), outputs=lambda _plan, _context, **_kwargs: (output,), ) - monkeypatch.setattr( - "openhcs.core.steps.function_outputs.runtime_artifact_materializations", - lambda _plan, _context: (materialization,), + saved_artifacts = ( + MaterializedRuntimeArtifact( + outputs_by_backend={Backend.DISK.value: (output,)}, + materialization=materialization, + ), ) output_plan = ArtifactOutputPlan( @@ -1769,6 +1788,7 @@ def test_runtime_multiplane_label_artifact_projects_persisted_source_binding( plan.artifact_outputs = {output_plan.ref(): output_plan} target = RuntimeArtifactMetadataTarget.from_plan(plan) assert target is not None + target = replace(target, artifact_materializations=saved_artifacts) [(projection, virtual_path)] = target.runtime_artifact_projection_paths( context, plan ) @@ -1933,10 +1953,12 @@ def test_produced_address_publication_never_parses_generated_filenames( output_dir = plate_root / "images" output_dir.mkdir(parents=True) parser = SourceSchemaFilenameParser() - filemanager = FileManager({ - Backend.DISK.value: DiskStorageBackend(), - Backend.MEMORY.value: MemoryStorageBackend(), - }) + filemanager = FileManager( + { + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + } + ) context = context_stub(filemanager, parser=parser) context.metadata_cache = { AllComponents.WELL: {well: None, "not-produced": None}, @@ -1962,12 +1984,17 @@ def test_produced_address_publication_never_parses_generated_filenames( metadata = ImagePayloadMetadata( source_path=source_path, source_component_metadata={ - "well": well, "site": 1, "channel": 1, - "z_index": z_index, "timepoint": 1, + "well": well, + "site": 1, + "channel": 1, + "z_index": z_index, + "timepoint": 1, }, source_voxel_spacing=SourceVoxelSpacing((0.5, 0.5)), ) - identity = FunctionOutputIdentityAuthority.identity_from_metadata(parser, metadata) + identity = FunctionOutputIdentityAuthority.identity_from_metadata( + parser, metadata + ) assert identity is not None assert identity.extension == extension identity = identity.with_filename_qualifier("centre_dots") @@ -1980,14 +2007,20 @@ def test_produced_address_publication_never_parses_generated_filenames( assert parsed.value_for(AllComponents.WELL) == well path = output_dir / filename tifffile.imwrite(path, pixels) - filemanager.save(metadata.payload_with(pixels, None), str(path), Backend.MEMORY.value) - records.append(ProducedOutputSemantics.from_output( - plan, path, identity, - output_context=AlignedImageSliceContext.main_flow( - output_key="centre_dots", artifact_kind=ImageArtifactType.value - ), - image_metadata=metadata, - )) + filemanager.save( + metadata.payload_with(pixels, None), str(path), Backend.MEMORY.value + ) + records.append( + ProducedOutputSemantics.from_output( + plan, + path, + identity, + output_context=AlignedImageSliceContext.main_flow( + output_key="centre_dots", artifact_kind=ImageArtifactType.value + ), + image_metadata=metadata, + ) + ) paths.append(f"images/{filename}") step_output_manifest(context).record_outputs(plan, records) @@ -2029,11 +2062,15 @@ def reject_generated_path_parse(_filename): removed = paths[0] (plate_root / removed).unlink() OpenHCSMetadataWriter.finalize_completed_plate({well: context}) - reconciled = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + reconciled = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES][ + "images" + ] assert set(reconciled[FIELDS.IMAGE_FILES]) == set(paths[1:]) assert removed not in reconciled[FIELDS.WORKSPACE_MAPPING] assert removed not in reconciled[FIELDS.SOURCE_METADATA] - assert {record["virtual_path"] for record in reconciled[FIELDS.SOURCE_PROJECTION]} == set(paths[1:]) + assert { + record["virtual_path"] for record in reconciled[FIELDS.SOURCE_PROJECTION] + } == set(paths[1:]) assert set(reconciled["z_indexes"]) == {str(z) for z in z_values[1:]} @@ -2044,8 +2081,7 @@ def test_produced_stacked_dotted_identity_keeps_declared_extension( ): parser = SourceSchemaFilenameParser() source_paths = tuple( - f"/source/{well}_s001_w1_z{z:03d}_t001{extension}" - for z in (3, 1, 2) + f"/source/{well}_s001_w1_z{z:03d}_t001{extension}" for z in (3, 1, 2) ) metadata = ImagePayloadMetadata( source_image_provenance_planes=SourceImageProvenancePlanes.from_components( @@ -2055,7 +2091,9 @@ def test_produced_stacked_dotted_identity_keeps_declared_extension( request = FunctionOutputPathRequest( parser=parser, output_dir=tmp_path, - output_payload=metadata.payload_with(np.zeros((3, 4, 5), dtype=np.uint16), None), + output_payload=metadata.payload_with( + np.zeros((3, 4, 5), dtype=np.uint16), None + ), input_path=Path(source_paths[0]).name, variable_components=(VariableComponents.Z_INDEX,), ) @@ -2176,7 +2214,9 @@ def test_runtime_image_metadata_target_requires_persisted_images(tmp_path, conte if contents == "tables": (directory / "measurements.csv").write_text("ObjectNumber,Area\n1,4\n") else: - tifffile.imwrite(directory / "A01_s001_w2_z001_t001.tif", np.ones((2, 2), dtype=np.uint8)) + tifffile.imwrite( + directory / "A01_s001_w2_z001_t001.tif", np.ones((2, 2), dtype=np.uint8) + ) context = context_stub(FileManager({Backend.DISK.value: DiskStorageBackend()})) plan = function_step_plan("SaveImages") plan.write_backend = Backend.MEMORY.value @@ -2223,7 +2263,32 @@ def test_runtime_image_metadata_target_requires_persisted_images(tmp_path, conte target = RuntimeArtifactMetadataTarget.from_plan(plan) assert target is not None assert target.output_dir == directory - selected = OpenHCSMetadataWriter.OutputTarget.for_execution(context, plan) + saved_artifacts = () + if contents == "images": + record = context.runtime_value_store.values()[0] + materialization = RuntimeArtifactMaterialization.from_record( + output_plan=output_plan, + record=record, + plan=plan, + context=context, + ) + saved_artifacts = ( + MaterializedRuntimeArtifact( + outputs_by_backend={ + Backend.DISK.value: ( + Output.from_metadata( + path=str(directory / "A01_s001_w2_z001_t001.tif"), + content=record.value.data, + metadata=image_payload_metadata(record.value.data), + ), + ) + }, + materialization=materialization, + ), + ) + selected = OpenHCSMetadataWriter.OutputTarget.for_execution( + context, plan, artifact_materializations=saved_artifacts + ) if contents == "images": assert selected == (target,) else: @@ -2295,8 +2360,12 @@ def test_declared_image_destinations_publish_and_reconcile_after_value_cleanup( path=output_plan.path, backend=Backend.MEMORY.value, ) - RuntimeArtifactMaterializationAuthority.materialize(context, plan) - OpenHCSMetadataWriter.write(context, plan) + materializations = RuntimeArtifactMaterializationAuthority.materialize( + context, plan + ) + OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ) context.runtime_value_store.clear() if stray_image: tifffile.imwrite(tmp_path / "images/review/unowned.tif", pixels) diff --git a/tests/unit/test_materialization_batch_outcomes.py b/tests/unit/test_materialization_batch_outcomes.py new file mode 100644 index 0000000000..a24b092c58 --- /dev/null +++ b/tests/unit/test_materialization_batch_outcomes.py @@ -0,0 +1,232 @@ +"""Actual batch outcomes, independent of subsequent logical-value rendering.""" + +from dataclasses import replace +from pathlib import Path +import pickle +from types import MappingProxyType + +import pytest +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.memory import MemoryStorageBackend + +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.compiled_step_plan import CompiledStepPlan +from openhcs.core.debug import NoOpDebugExecutionPolicy +from openhcs.core.function_patterns import compile_function_pattern +from openhcs.core.orchestrator.compiled_plate_execution import ( + CompiledPlateExecutionExtras, + CompiledPlateExecutionResults, +) +from openhcs.core.orchestrator.execution_result import ( + ExecutionResult, + RuntimeContextObservation, + RuntimeExecutionObservation, + RuntimeObservationMode, +) +from openhcs.core.orchestrator.worker_execution import ( + _execute_axis_with_sequential_combinations, +) +from openhcs.core.orchestrator.worker_lanes import WorkerLaneExecutionContext +from openhcs.core.runtime_exports import RuntimeExportObservation +from openhcs.core.steps.abstract import StepExecutionObservation +from openhcs.core.steps.function_step import FunctionStep +from openhcs.processing.materialization import ( + CsvOptions, + JsonOptions, + MaterializationSpec, + SavedMaterializationOutputs, + prepare_materialization, +) +from openhcs.processing.materialization.core import _WRITERS_BY_OPTIONS + + +class CsvOnlyMemoryBackend(MemoryStorageBackend): + def accepts_payload(self, data, path): + return Path(path).suffix == ".csv" + + +def test_rendered_batch_reuses_exact_outputs_after_logical_payload_mutates( + tmp_path, monkeypatch +): + manager = FileManager({"disk": DiskStorageBackend()}) + rows = [{"Area": 42}] + batch = prepare_materialization( + MaterializationSpec(CsvOptions(), JsonOptions()), + rows, + str(tmp_path / "measurements"), + manager, + ("disk",), + ) + original_outputs = batch.outputs + rows[0]["Area"] = 999 + + def render_again(*_args): + pytest.fail( + "Saving a prepared batch must not render mutable logical input again" + ) + + for option_type in (CsvOptions, JsonOptions): + monkeypatch.setitem( + _WRITERS_BY_OPTIONS, + option_type, + replace(_WRITERS_BY_OPTIONS[option_type], write=render_again), + ) + outcome = batch.save() + + assert isinstance(outcome, SavedMaterializationOutputs) + assert outcome.outputs_for_backend("disk") == original_outputs + assert all( + left is right + for left, right in zip( + outcome.outputs_for_backend("disk"), original_outputs, strict=True + ) + ) + assert Path(batch.primary_path).read_text().splitlines() == ["Area", "42"] + assert Path(original_outputs[1].path).read_text().find("999") == -1 + + +def test_saved_outcomes_include_only_backend_accepted_outputs(tmp_path): + manager = FileManager({"memory": CsvOnlyMemoryBackend()}) + batch = prepare_materialization( + MaterializationSpec(CsvOptions(), JsonOptions()), + [{"Area": 42}], + str(tmp_path / "measurements"), + manager, + ("memory",), + ) + + outcome = batch.save() + + assert outcome.outputs_for_backend("memory") == (batch.outputs[0],) + assert outcome.outputs_for_backend("disk") == () + assert manager.exists(batch.outputs[0].path, "memory") + assert not manager.exists(batch.outputs[1].path, "memory") + + +def test_failed_save_does_not_return_a_successful_outcome(tmp_path, monkeypatch): + manager = FileManager({"disk": DiskStorageBackend()}) + batch = prepare_materialization( + MaterializationSpec(CsvOptions()), + [{"Area": 42}], + str(tmp_path / "measurements"), + manager, + ("disk",), + ) + + def failed_save(*_args, **_kwargs): + raise OSError("destination is unavailable") + + monkeypatch.setattr(manager, "save_batch", failed_save) + with pytest.raises(OSError, match="destination is unavailable"): + batch.save() + assert not Path(batch.primary_path).exists() + + +class EmittedOutputStep(FunctionStep): + def __init__(self, path): + super().__init__(lambda image: image) + self.path = path + + def process(self, context, step_index): + self.path.write_text("Area\n42\n") + return StepExecutionObservation(MappingProxyType({}), (self.path,)) + + +class FrozenWorkerContext(ProcessingContext): + def __init__(self): + super().__init__( + axis_id="A01", + filemanager=FileManager({"memory": MemoryStorageBackend()}), + step_plans={ + 0: CompiledStepPlan( + step_index=0, + step_name="Save", + step_type="FunctionStep", + axis_id="A01", + compiled_function_pattern=compile_function_pattern( + lambda image: image, {}, {} + ), + ) + }, + ) + self.freeze() + self.released = False + + def release_execution_image_cache(self): + super().release_execution_image_cache() + self.released = True + + +@pytest.mark.parametrize("mode", tuple(RuntimeObservationMode)) +def test_worker_exports_actual_step_outcomes_after_resource_release( + tmp_path, monkeypatch, mode +): + import openhcs.core.orchestrator.worker_execution as worker + + path = tmp_path / "actual.csv" + context = FrozenWorkerContext() + lane = WorkerLaneExecutionContext( + execution_id="execution", + plate_id="plate", + debug_execution_policy=NoOpDebugExecutionPolicy(), + worker_slot="worker-0", + worker_assignments={"worker-0": ("A01",)}, + ) + monkeypatch.setattr(worker, "emit", lambda **_kwargs: None) + + def forbidden_reconstruction(*_args, **_kwargs): + pytest.fail("Ordinary execution must consume actual step outcomes") + + monkeypatch.setattr( + worker, + "preview_reused_materialized_artifact_locations", + forbidden_reconstruction, + ) + monkeypatch.setattr( + worker, + "preview_reused_runtime_export_paths", + forbidden_reconstruction, + ) + result = _execute_axis_with_sequential_combinations( + [EmittedOutputStep(path)], + [("A01", context)], + lane, + mode, + release_axis_resources=False, + ) + transported = pickle.loads(pickle.dumps(result)) + + assert context.released + assert transported.is_success() + assert transported.runtime_observation.contexts[0].runtime_export_paths == (path,) + assert transported.runtime_observation.contexts[0].records == () + assert RuntimeExportObservation.from_runtime_observations( + (transported.runtime_observation,) + ).table_outputs == (path,) + + +def test_parent_and_worker_exports_share_existing_observation_authority(tmp_path): + worker_path = tmp_path / "worker.csv" + plate_path = tmp_path / "plate.csv" + unrelated = tmp_path / "unrelated.csv" + for path in (worker_path, plate_path, unrelated): + path.write_text("Area\n42\n") + worker = RuntimeExecutionObservation( + contexts=(RuntimeContextObservation("A01", (), (worker_path,)),) + ) + parent = RuntimeExecutionObservation( + contexts=(RuntimeContextObservation("A01", (), (plate_path,)),) + ) + results = CompiledPlateExecutionResults( + {"A01": ExecutionResult.success("A01", runtime_observation=worker)}, + extras=CompiledPlateExecutionExtras( + viewer_states_by_port={}, runtime_observation=parent + ), + ) + exports = RuntimeExportObservation.from_runtime_observations( + results.runtime_observations + ) + + assert exports.output_files == (worker_path, plate_path) + assert unrelated not in exports.output_files diff --git a/tests/unit/test_runtime_execution_validation.py b/tests/unit/test_runtime_execution_validation.py index 077d4a5e41..9906fed3bb 100644 --- a/tests/unit/test_runtime_execution_validation.py +++ b/tests/unit/test_runtime_execution_validation.py @@ -33,7 +33,11 @@ from openhcs.core.measurement_row_materialization import ( MeasurementSparseColumnarRows, ) -from openhcs.core.orchestrator.execution_result import ExecutionResult +from openhcs.core.orchestrator.execution_result import ( + ExecutionResult, + RuntimeContextObservation, + RuntimeExecutionObservation, +) from openhcs.core.pipeline.function_contracts import execution_scope from openhcs.core.runtime_artifact_values import ( ArtifactKey, @@ -120,7 +124,8 @@ def _streaming_context( def test_runtime_execution_validation_detects_missing_artifact_kind() -> None: observation = RuntimeArtifactExecutionObservation.from_contexts( - {"A01": ProcessingContext(axis_id="A01")} + {"A01": ProcessingContext(axis_id="A01")}, + runtime_observations=(), ) failures = runtime_artifact_execution_failures( @@ -186,7 +191,9 @@ def test_runtime_execution_observation_reads_context_stores() -> None: backend="memory", ) - observation = RuntimeArtifactExecutionObservation.from_contexts({"A01": context}) + observation = RuntimeArtifactExecutionObservation.from_contexts( + {"A01": context}, runtime_observations=() + ) assert observation.record_counts_by_axis["A01"][MeasurementsArtifactType] == 1 @@ -204,7 +211,9 @@ def test_runtime_execution_observation_ignores_uncontracted_files( context = ProcessingContext(axis_id="A01") - observation = RuntimeArtifactExecutionObservation.from_contexts({"A01": context}) + observation = RuntimeArtifactExecutionObservation.from_contexts( + {"A01": context}, runtime_observations=() + ) assert observation.exports.table_outputs == () assert observation.exports.image_outputs == () @@ -239,16 +248,23 @@ def test_zmq_observation_exports_exact_compiler_owned_artifacts( }, axis_id="A01", ) - monkeypatch.setattr( - "openhcs.core.steps.function_artifact_materialization." - "runtime_export_artifact_output_paths", - lambda _plan, _context: (contracted_output,), + runtime_observations = ( + RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + context_key="A01", + records=(), + runtime_export_paths=(contracted_output,), + ), + ) + ), ) export = ZMQRuntimeExecutionObservationExport.from_execution( compiled_contexts={"A01": context}, execution_results={"A01": ExecutionResult.success("A01")}, output_roots=(tmp_path,), + runtime_observations=runtime_observations, ) assert export.expectation.artifact_kinds == frozenset((MeasurementsArtifactType,)) @@ -258,7 +274,9 @@ def test_zmq_observation_exports_exact_compiler_owned_artifacts( assert export.exports.output_files == (contracted_output,) assert unrelated_output not in export.exports.output_files assert ( - RuntimeExportObservation.from_execution_contexts({"A01": context}).output_files + RuntimeExportObservation.from_runtime_observations( + runtime_observations + ).output_files == export.exports.output_files ) @@ -514,22 +532,22 @@ def test_runtime_execution_observation_reads_plate_export_from_exact_owner( ) for axis_id, owns_output in (("A01", True), ("A02", False)) } - observed_axes: list[str] = [] - - def output_paths(plan, _context): - observed_axes.append(plan.axis_id) - return (contracted_output,) - - monkeypatch.setattr( - "openhcs.core.steps.function_artifact_materialization." - "runtime_export_artifact_output_paths", - output_paths, + runtime_observations = ( + RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + context_key="A01", + records=(), + runtime_export_paths=(contracted_output,), + ), + ) + ), + ) + observation = RuntimeArtifactExecutionObservation.from_contexts( + contexts, runtime_observations=runtime_observations ) - - observation = RuntimeArtifactExecutionObservation.from_contexts(contexts) expectation = RuntimeArtifactExecutionExpectation.from_compiled_contexts(contexts) - assert observed_axes == ["A01"] assert observation.exports.table_outputs == (contracted_output,) assert expectation.artifact_kinds == frozenset((MeasurementsArtifactType,)) assert tuple( @@ -578,6 +596,7 @@ def test_zmq_observation_compresses_and_preserves_exact_runtime_records( compiled_contexts={"A01": context}, execution_results={"A01": ExecutionResult.success("A01")}, output_roots=(tmp_path,), + runtime_observations=(), ) path = tmp_path / "observation.pkl" diff --git a/tests/unit/test_zmq_execution_server.py b/tests/unit/test_zmq_execution_server.py index 45e8070fa3..85599621bf 100644 --- a/tests/unit/test_zmq_execution_server.py +++ b/tests/unit/test_zmq_execution_server.py @@ -16,6 +16,9 @@ ProcessingConfig, ) from openhcs.core.execution_state import ExecutionOutputPlateSummary +from openhcs.core.orchestrator.compiled_plate_execution import ( + CompiledPlateExecutionResults, +) from openhcs.core.orchestrator.execution_result import ( ExecutionResult, RuntimeContextObservation, @@ -196,20 +199,22 @@ def test_server_exports_outcomes_without_projecting_compiled_values( server._export_runtime_observation( request_context=request_context, compilation=compilation, - execution_results={ - "A01": ExecutionResult.success( - "A01", - runtime_observation=RuntimeExecutionObservation( - contexts=( - RuntimeContextObservation( - "context", - (), - runtime_export_paths=(declared_output,), - ), - ) - ), - ) - }, + execution_results=CompiledPlateExecutionResults( + { + "A01": ExecutionResult.success( + "A01", + runtime_observation=RuntimeExecutionObservation( + contexts=( + RuntimeContextObservation( + "context", + (), + runtime_export_paths=(declared_output,), + ), + ) + ), + ) + } + ), ) export = ZMQRuntimeExecutionOutcomeExport.read(export_path) diff --git a/tests/unit/test_zmq_runtime_outcome_export.py b/tests/unit/test_zmq_runtime_outcome_export.py index f358c89ca1..041116b678 100644 --- a/tests/unit/test_zmq_runtime_outcome_export.py +++ b/tests/unit/test_zmq_runtime_outcome_export.py @@ -110,6 +110,7 @@ def test_value_export_rejects_an_uncompiled_execution_axis() -> None: compiled_contexts={}, execution_results={"A01": ExecutionResult.success("A01")}, output_roots=(), + runtime_observations=(), ) with pytest.raises(RuntimeError, match="execution outcomes have no compiled axis"): exported.require_valid_observation() @@ -167,6 +168,7 @@ def test_previous_runtime_export_versions_remain_readable(tmp_path: Path) -> Non execution_results={}, output_roots=(), execution_id="new-job", + runtime_observations=(), ), schema_version=6, execution_id=None, @@ -178,6 +180,7 @@ def test_previous_runtime_export_versions_remain_readable(tmp_path: Path) -> Non execution_results={}, output_roots=(), execution_id="other-job", + runtime_observations=(), ).write(observation_path) assert observation_path.read_bytes() == retained_bytes assert ( From e4d786cbf6a688d9543213bc314cac5a3717bb97 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:09:54 -0400 Subject: [PATCH 005/125] Resolve runtime source metadata through nominal record lifetimes --- openhcs/core/runtime_adapters.py | 19 +- openhcs/core/runtime_source_binding_cache.py | 102 ++++++- openhcs/core/source_binding_selection.py | 134 ++++++--- openhcs/core/steps/function_execution.py | 9 +- openhcs/core/steps/function_runtime.py | 2 +- ...test_function_runtime_source_projection.py | 2 + ...test_runtime_source_resolution_snapshot.py | 276 ++++++++++++++++++ 7 files changed, 498 insertions(+), 46 deletions(-) create mode 100644 tests/unit/test_runtime_source_resolution_snapshot.py diff --git a/openhcs/core/runtime_adapters.py b/openhcs/core/runtime_adapters.py index 14e0bb43cc..e8b2d3cf2a 100644 --- a/openhcs/core/runtime_adapters.py +++ b/openhcs/core/runtime_adapters.py @@ -42,7 +42,6 @@ ) from openhcs.core.source_binding_selection import ( SourceBindingMatchedImageSet, - SourcePatternResolutionContext, ) from openhcs.core.source_image_provenance import ( SourceImageIdentity, @@ -305,10 +304,12 @@ def source_artifact_payload(self, ref: ArtifactSpecRef) -> object: projection, axis_id=self.axis_scope.axis_id, ) - source_context = SourcePatternResolutionContext.from_projection( - parser=self.context.microscope_handler.parser, - projection=projection, - metadata_rules=self.source_binding_plan.metadata_rules, + source_context = ( + self.context.runtime_source_binding_context_cache.source_pattern_context( + parser=self.context.microscope_handler.parser, + projection=projection, + metadata_rules=self.source_binding_plan.metadata_rules, + ) ) matched_set = SourceBindingMatchedImageSet.from_plan( bindings=self.source_binding_plan.binding_declarations, @@ -475,7 +476,9 @@ class RuntimeAdapterSpec: parameter_name: str factory: RuntimeAdapterFactory manages_artifact_inputs: bool = False - artifact_output_policy: type[ArtifactOutputPolicy] = NativeReturnArtifactOutputPolicy + artifact_output_policy: type[ArtifactOutputPolicy] = ( + NativeReturnArtifactOutputPolicy + ) runtime_callable_factory: RuntimeCallableFactory | None = None def __post_init__(self) -> None: @@ -530,7 +533,9 @@ def runtime_adapter( factory: RuntimeAdapterFactory, *, manages_artifact_inputs: bool = False, - artifact_output_policy: type[ArtifactOutputPolicy] = NativeReturnArtifactOutputPolicy, + artifact_output_policy: type[ + ArtifactOutputPolicy + ] = NativeReturnArtifactOutputPolicy, runtime_callable_factory: RuntimeCallableFactory | None = None, ) -> Callable[[_F], _F]: """Declare that a callable needs an invocation-scoped runtime adapter.""" diff --git a/openhcs/core/runtime_source_binding_cache.py b/openhcs/core/runtime_source_binding_cache.py index 7e2ffb242d..d067f2b5fe 100644 --- a/openhcs/core/runtime_source_binding_cache.py +++ b/openhcs/core/runtime_source_binding_cache.py @@ -3,17 +3,85 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any, TYPE_CHECKING +from dataclasses import dataclass, field, replace +from types import MappingProxyType +from typing import Any, Hashable, TYPE_CHECKING from openhcs.core.source_bindings import ( SourceBindingRuntimeContext, SourceBindingRuntimeMetadataNormalizer, + MetadataExtractionRule, ) from openhcs.core.source_metadata import SourceMetadataMapping if TYPE_CHECKING: - from openhcs.core.source_binding_selection import SourceUniverseRuntimeState + from openhcs.core.source_binding_selection import ( + SourcePatternResolutionContext, + SourceUniverseRuntimeState, + ) + from openhcs.core.source_workspace_projection import ( + VirtualWorkspaceSourceProjection, + ) + from openhcs.microscopes.microscope_interfaces import FilenameParser + + +@dataclass(frozen=True, slots=True) +class RuntimeSourceResolutionSnapshot: + """A derived resolution view retaining its exact projection and parser owners.""" + + projection: "VirtualWorkspaceSourceProjection" + context: "SourcePatternResolutionContext" + + @classmethod + def from_projection( + cls, + *, + parser: "FilenameParser", + projection: "VirtualWorkspaceSourceProjection", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> "RuntimeSourceResolutionSnapshot": + """Resolve declared positions into one independently owned runtime view.""" + from openhcs.core.source_binding_selection import ( + ResolvedSourceMetadataRecord, + SourceMetadataRecord, + SourcePatternResolutionContext, + ) + + context = SourcePatternResolutionContext.from_projection( + parser=parser, + projection=projection, + metadata_rules=metadata_rules, + ) + normalized_metadata = SourceBindingRuntimeMetadataNormalizer( + projection.source_metadata_by_path + ).normalized() + context = replace( + context, + source_metadata_by_path=MappingProxyType( + { + path: SourceMetadataRecord.from_mapping(metadata) + for path, metadata in normalized_metadata.items() + } + ), + source_projections_by_virtual_path=MappingProxyType( + dict(context.source_projections_by_virtual_path) + ), + ) + paths = dict.fromkeys( + ( + *context.source_metadata_by_path, + *context.source_paths_by_virtual_path, + *context.source_paths_by_virtual_path.values(), + ) + ) + records = {} + for path in paths: + metadata = context.metadata_for_path(path) + records[path] = ResolvedSourceMetadataRecord.from_resolved_mapping( + {} if metadata is None else metadata + ) + context = replace(context, source_metadata_by_path=MappingProxyType(records)) + return cls(projection=projection, context=context) @dataclass(slots=True) @@ -32,6 +100,34 @@ class RuntimeSourceBindingContextCache: tuple[int, tuple[str, ...], object, int | None], "SourceUniverseRuntimeState", ] = field(default_factory=dict) + source_resolution_snapshots: dict[ + tuple[int, int, tuple[Hashable, ...], tuple[MetadataExtractionRule, ...]], + RuntimeSourceResolutionSnapshot, + ] = field(default_factory=dict) + + def source_pattern_context( + self, + *, + parser: "FilenameParser", + projection: "VirtualWorkspaceSourceProjection", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> "SourcePatternResolutionContext": + """Own one normalized runtime snapshot for exact source declarations.""" + key = (id(projection), id(parser), parser.semantic_identity(), metadata_rules) + cached = self.source_resolution_snapshots.get(key) + if cached is not None: + return cached.context + snapshot = RuntimeSourceResolutionSnapshot.from_projection( + parser=parser, + projection=projection, + metadata_rules=metadata_rules, + ) + self.source_resolution_snapshots[key] = snapshot + return snapshot.context + + def __reduce__(self) -> tuple[type["RuntimeSourceBindingContextCache"], tuple[()]]: + """Transport reconstructs all derived caches from declaration defaults.""" + return type(self), () def normalized_source_metadata( self, diff --git a/openhcs/core/source_binding_selection.py b/openhcs/core/source_binding_selection.py index 90c686c60a..13e7673ba9 100644 --- a/openhcs/core/source_binding_selection.py +++ b/openhcs/core/source_binding_selection.py @@ -26,6 +26,7 @@ SourceBindingMatchMethod, SourceBindingMatchPlan, SourceBindingRuntimeContext, + SourceBindingRuntimeMetadataNormalizer, SourceSetRole, ) from openhcs.core.source_image_provenance import ( @@ -77,7 +78,7 @@ def _cached_source_candidate_pattern_keys(pattern_path: str) -> tuple[str, ...]: return tuple(dict.fromkeys((pattern_path, path.as_posix(), path.name))) -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, eq=False) class SourceMetadataRecord(Mapping[str, SourceMetadataValue]): """Normalized source metadata carried across source-binding selection.""" @@ -85,7 +86,51 @@ class SourceMetadataRecord(Mapping[str, SourceMetadataValue]): @classmethod def from_mapping(cls, metadata: SourceMetadataMapping) -> "SourceMetadataRecord": - return cls(tuple((str(key), value) for key, value in metadata.items())) + return DeclaredSourceMetadataRecord( + tuple((str(key), value) for key, value in metadata.items()) + ) + + @abstractmethod + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> "SourceMetadataRecord | None": + """Return this path's metadata through its declared resolution lifetime.""" + + def resolve_with_fallback( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> "SourceMetadataRecord | None": + """Fill parser and extraction-rule fields absent from declarations.""" + metadata: dict[str, SourceMetadataValue] = {} + merge_source_metadata(metadata, self, path=path) + parsed_metadata = parser.parse_filename(path) + if parsed_metadata is not None: + merge_source_metadata( + metadata, + { + key: value + for key, value in parsed_metadata.wire_mapping().items() + if key not in metadata + }, + path=path, + ) + rule_metadata = metadata_from_rules(path, metadata_rules) + if rule_metadata: + merge_source_metadata( + metadata, + { + key: value + for key, value in rule_metadata.items() + if key not in metadata + }, + path=path, + ) + return self.from_mapping(metadata) if metadata else None def __getitem__(self, key: str) -> SourceMetadataValue: for field_key, value in self.fields: @@ -99,6 +144,54 @@ def __iter__(self) -> Iterator[str]: def __len__(self) -> int: return len(self.fields) + def __eq__(self, other: object) -> bool: + if not isinstance(other, SourceMetadataRecord): + return NotImplemented + return self.fields == other.fields + + def __hash__(self) -> int: + return hash(self.fields) + + +@dataclass(frozen=True, slots=True, eq=False) +class DeclaredSourceMetadataRecord(SourceMetadataRecord): + """Live declared metadata that still requires path-specific fallbacks.""" + + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> SourceMetadataRecord | None: + return self.resolve_with_fallback(path, parser, metadata_rules) + + +@dataclass(frozen=True, slots=True, eq=False) +class ResolvedSourceMetadataRecord(SourceMetadataRecord): + """Immutable path-specific metadata resolved at a runtime snapshot boundary.""" + + @classmethod + def from_resolved_mapping( + cls, metadata: SourceMetadataMapping + ) -> "ResolvedSourceMetadataRecord": + return cls( + tuple( + ( + str(key), + SourceBindingRuntimeMetadataNormalizer.normalized_value(value), + ) + for key, value in metadata.items() + ) + ) + + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple[MetadataExtractionRule, ...], + ) -> SourceMetadataRecord | None: + return self if self.fields else None + @dataclass(frozen=True, slots=True) class SourceMetadataCandidates: @@ -404,35 +497,10 @@ def metadata_for_paths( ) def metadata_for_path(self, path: str) -> SourceMetadataRecord | None: - metadata: dict[str, SourceMetadataValue] = {} - declared_metadata = self.source_metadata_by_path.get(path) - if declared_metadata is not None: - merge_source_metadata(metadata, declared_metadata, path=path) - parsed_metadata = self.parser.parse_filename(path) - if parsed_metadata is not None: - merge_source_metadata( - metadata, - { - key: value - for key, value in parsed_metadata.wire_mapping().items() - if key not in metadata - }, - path=path, - ) - rule_metadata = metadata_from_rules(path, self.metadata_rules) - if rule_metadata: - merge_source_metadata( - metadata, - { - key: value - for key, value in rule_metadata.items() - if key not in metadata - }, - path=path, - ) - if metadata: - return SourceMetadataRecord.from_mapping(metadata) - return None + record = self.source_metadata_by_path.get(path) + if record is None: + record = DeclaredSourceMetadataRecord(()) + return record.resolve(path, self.parser, self.metadata_rules) def merged_metadata_for_paths( self, @@ -1369,7 +1437,7 @@ def _source_image_set_identity( metadata = ( metadata_candidates.values[0] if metadata_candidates.values - else SourceMetadataRecord(()) + else DeclaredSourceMetadataRecord(()) ) return SourceImageSetIdentity.from_metadata( metadata, @@ -1708,7 +1776,7 @@ def source_metadata_by_path(self) -> Mapping[str, SourceMetadataMapping]: def source_context(self) -> SourcePatternResolutionContext: metadata_rules = self.plan.source_binding_plan.metadata_rules if self.source_projection is not None: - return SourcePatternResolutionContext.from_projection( + return self.context.runtime_source_binding_context_cache.source_pattern_context( parser=self.context.microscope_handler.parser, projection=self.source_projection, metadata_rules=metadata_rules, diff --git a/openhcs/core/steps/function_execution.py b/openhcs/core/steps/function_execution.py index 1b075b9d82..6cd8cf743c 100644 --- a/openhcs/core/steps/function_execution.py +++ b/openhcs/core/steps/function_execution.py @@ -7,7 +7,7 @@ import time import traceback from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from itertools import zip_longest from typing import TYPE_CHECKING @@ -28,6 +28,7 @@ ) from openhcs.core.progress import ProgressPhase, ProgressStatus, emit from openhcs.core.runtime_pattern_cache import RuntimePatternDiscoveryCacheKey +from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache from openhcs.core.source_binding_selection import ( SourceBoundAnchorPatternPolicy, SourceCandidatePath, @@ -223,6 +224,9 @@ class StepAnchorPatternFilter: output_manifest: StepOutputManifestStore source_workspace_authority: VirtualWorkspaceSourceProjectionAuthority source_workspace_projection_cache: VirtualWorkspaceSourceProjectionCache + source_binding_context_cache: RuntimeSourceBindingContextCache = field( + default_factory=RuntimeSourceBindingContextCache + ) @classmethod def from_context( @@ -243,6 +247,7 @@ def from_context( source_workspace_projection_cache=( context.runtime_source_workspace_projection_cache ), + source_binding_context_cache=context.runtime_source_binding_context_cache, ) def filtered(self, grouped_patterns: PatternGroups) -> PatternGroups: @@ -611,7 +616,7 @@ def source_pattern_context(self) -> SourcePatternResolutionContext: """Return source-path context used to filter source-bound anchors.""" projection = self.source_workspace_authority.projection_or_empty() - return SourcePatternResolutionContext.from_projection( + return self.source_binding_context_cache.source_pattern_context( parser=self.parser, projection=self.source_workspace_projection_cache.filtered_by_axis( projection, diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index e5fb4cf954..266ed9b128 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -2948,7 +2948,7 @@ def _source_binding_load_universe(self) -> tuple[str, ...]: def _source_binding_candidate_context(self) -> SourcePatternResolutionContext: projection = self.source_workspace_projection_authority().projection_or_empty() - return SourcePatternResolutionContext.from_projection( + return self.request.context.runtime_source_binding_context_cache.source_pattern_context( parser=self.request.context.microscope_handler.parser, projection=self.source_workspace_projection_cache().filtered_by_axis( projection, diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index d031997b88..016ae53eaf 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -3733,6 +3733,7 @@ def test_producer_anchored_pipeline_start_paths_use_exact_source_projection_bund """Producer bookkeeping must not override exact workspace source ownership.""" from openhcs.core.steps import function_runtime + from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache virtual_paths = ( "A01_s001_w1_z001_t001.tif", @@ -3870,6 +3871,7 @@ def resolve_address(backend_address, backend, *, base_path): ), filemanager=SourceFileManager(), runtime_image_stack_cache=RuntimeImageStackCache(), + runtime_source_binding_context_cache=RuntimeSourceBindingContextCache(), runtime_source_workspace_projection_cache=( VirtualWorkspaceSourceProjectionCache() ), diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py new file mode 100644 index 0000000000..9033f38827 --- /dev/null +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -0,0 +1,276 @@ +"""Resolution lifetimes across generic source selectors and artifact adapters.""" + +import pickle +from types import MappingProxyType + +import pytest +from polystore.virtual_workspace import SourcePixelRef + +from openhcs.constants.constants import AllComponents +from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache +from openhcs.core.source_binding_selection import ( + DeclaredSourceMetadataRecord, + ResolvedSourceMetadataRecord, + SourceBindingMatchedImageSet, + SourceMetadataRecord, + SourcePatternResolutionContext, +) +from openhcs.core.source_bindings import ( + MetadataExtractionRule, + MetadataSource, + SourceBindingRuntimeContext, +) +from openhcs.core.source_matching import SourceImageSetIdentityPolicy +from openhcs.core.source_metadata import ORIGINAL_SOURCE_METADATA_FIELD +from openhcs.core.source_projection import OpenHCSPlaneAddress, SourcePlaneProjection +from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection +from openhcs.microscopes.microscope_interfaces import FilenameParseResult +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser + +PATH = "A01_s001_w1_z001_t001.tif" + + +class CountingParser(SourceSchemaFilenameParser): + def __init__(self): + super().__init__() + self.calls = [] + + def parse_filename(self, path): + self.calls.append(path) + result = super().parse_filename(path) + if result is None or self.pattern_format is None: + return result + return FilenameParseResult( + tuple( + ( + component, + ( + int(self.pattern_format) + if component is AllComponents.SITE + else value + ), + ) + for component, value in result.declared_values() + ), + extension=result.extension, + ) + + +def projection(metadata): + return VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path={PATH: SourcePixelRef("disk", "/source/image.tif")}, + source_metadata_by_path={PATH: metadata}, + ) + + +def snapshot(cache, source_projection, parser, rules=()): + return cache.source_pattern_context( + parser=parser, projection=source_projection, metadata_rules=rules + ) + + +def test_declared_record_resolves_live_parser_and_nested_metadata(): + nested = {"Plate": "before"} + record = SourceMetadataRecord.from_mapping( + {ORIGINAL_SOURCE_METADATA_FIELD: nested, "site": 8} + ) + assert isinstance(record, DeclaredSourceMetadataRecord) + parser = CountingParser() + first = record.resolve(PATH, parser, ()) + nested["Plate"] = "after" + second = record.resolve(PATH, parser, ()) + assert first["site"] == second["site"] == 8 + assert first[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "before" + assert second[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "after" + assert parser.calls == [PATH, PATH] + + +def test_runtime_snapshot_owns_nested_metadata_and_unknown_fallback(): + nested = {"Plate": "before"} + source_projection = projection({ORIGINAL_SOURCE_METADATA_FIELD: nested}) + parser = CountingParser() + cache = RuntimeSourceBindingContextCache() + context = snapshot(cache, source_projection, parser) + calls = tuple(parser.calls) + nested["Plate"] = "after" + for _ in range(3): + metadata = context.metadata_for_path(PATH) + assert isinstance(metadata, ResolvedSourceMetadataRecord) + assert metadata[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "before" + assert tuple(parser.calls) == calls + with pytest.raises(TypeError): + metadata[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] = "invalid" + unknown = "A02_s001_w1_z001_t001.tif" + assert context.metadata_for_path(unknown)["well"] == "A02" + assert parser.calls[-1] == unknown + assert snapshot(cache, source_projection, parser) is context + entry = next(iter(cache.source_resolution_snapshots.values())) + assert entry.projection is source_projection + assert entry.context.parser is parser + + +def test_direct_projection_context_remains_live(): + nested = {"Plate": "before"} + parser = CountingParser() + context = SourcePatternResolutionContext.from_projection( + parser=parser, + projection=projection({ORIGINAL_SOURCE_METADATA_FIELD: nested}), + ) + assert ( + context.metadata_for_path(PATH)[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] + == "before" + ) + nested["Plate"] = "after" + assert ( + context.metadata_for_path(PATH)[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] + == "after" + ) + assert parser.calls == [PATH, PATH] + + +def test_snapshot_replacement_tracks_projection_parser_semantics_and_rules(): + cache = RuntimeSourceBindingContextCache() + parser = CountingParser() + source_projection = projection({}) + original = snapshot(cache, source_projection, parser) + parser.pattern_format = "9" + changed_parser = snapshot(cache, source_projection, parser) + assert changed_parser is not original + assert original.metadata_for_path(PATH)["site"] == 1 + assert changed_parser.metadata_for_path(PATH)["site"] == 9 + replaced_projection = snapshot(cache, projection({"site": 7}), parser) + assert replaced_projection.metadata_for_path(PATH)["site"] == 7 + replaced_parser = snapshot(cache, source_projection, CountingParser()) + assert replaced_parser is not original + assert replaced_parser.metadata_for_path(PATH)["site"] == 1 + rules = ( + MetadataExtractionRule( + source=MetadataSource.FILE_NAME, pattern=r"(?PA01)" + ), + ) + changed_rules = snapshot(cache, source_projection, parser, rules) + assert changed_rules is not changed_parser + assert ( + changed_rules.metadata_for_path(PATH)[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] + == "A01" + ) + + +def test_matched_set_preserves_resolved_record_behavior_and_precedence(): + parser = CountingParser() + context = snapshot( + RuntimeSourceBindingContextCache(), projection({"site": None}), parser + ) + matched = SourceBindingMatchedImageSet.from_plan( + bindings=(), + match_plan=None, + source_context=context, + identity_policy=SourceImageSetIdentityPolicy(), + ) + calls = tuple(parser.calls) + assert ( + matched.source_metadata_by_path[PATH] is context.source_metadata_by_path[PATH] + ) + assert matched.metadata_for_path(PATH)["site"] == 1 + assert tuple(parser.calls) == calls + + +def test_virtual_positions_remain_distinct_with_shared_physical_address(): + other = "A01_s002_w1_z001_t001.tif" + source_projection = VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path={ + PATH: SourcePixelRef("disk", "/source/shared.tif"), + other: SourcePixelRef("disk", "/source/shared.tif"), + }, + source_metadata_by_path=MappingProxyType( + {PATH: {"site": 1}, other: {"site": 2}} + ), + ) + context = snapshot( + RuntimeSourceBindingContextCache(), source_projection, CountingParser() + ) + assert context.virtual_paths_for_source("/source/shared.tif") == (PATH, other) + assert context.metadata_for_path(PATH)["site"] == 1 + assert context.metadata_for_path(other)["site"] == 2 + + +def test_known_empty_record_remains_none_without_repeated_parsing(): + parser = CountingParser() + source_projection = VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path={"opaque": SourcePixelRef("disk", "opaque")}, + source_metadata_by_path={}, + ) + context = snapshot(RuntimeSourceBindingContextCache(), source_projection, parser) + calls = tuple(parser.calls) + assert context.metadata_for_path("opaque") is None + assert context.metadata_for_path("opaque") is None + assert tuple(parser.calls) == calls + + +def test_snapshot_rejects_nested_values_outside_existing_metadata_grammar(): + with pytest.raises(TypeError, match="Source metadata scalar values"): + snapshot( + RuntimeSourceBindingContextCache(), + projection({"nested": {"unsupported": {"deep": 1}}}), + CountingParser(), + ) + + +def test_declared_and_resolved_records_preserve_family_value_identity(): + declared = SourceMetadataRecord.from_mapping({"site": 1, "well": "A01"}) + resolved = ResolvedSourceMetadataRecord.from_resolved_mapping( + {"site": 1, "well": "A01"} + ) + assert declared == resolved + assert hash(declared) == hash(resolved) + assert declared != SourceMetadataRecord.from_mapping({"site": 2, "well": "A01"}) + nested = {ORIGINAL_SOURCE_METADATA_FIELD: {"Plate": "A"}} + with pytest.raises(TypeError): + hash(SourceMetadataRecord.from_mapping(nested)) + with pytest.raises(TypeError): + hash(ResolvedSourceMetadataRecord.from_resolved_mapping(nested)) + + +def test_snapshot_owns_position_and_projection_map_views(): + original_ref = SourcePixelRef("disk", "/source/original.tif") + refs = {PATH: original_ref} + positions = { + PATH: SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values("A01", 1, 1, 1, 1), + ref=original_ref, + source_alias="original", + ) + } + original_projection = positions[PATH] + source_projection = VirtualWorkspaceSourceProjection( + source_refs_by_virtual_path=refs, + source_metadata_by_path={PATH: {}}, + source_projections_by_virtual_path=positions, + ) + context = snapshot( + RuntimeSourceBindingContextCache(), source_projection, CountingParser() + ) + refs[PATH] = SourcePixelRef("disk", "/source/replaced.tif") + positions.clear() + assert context.source_paths_by_virtual_path[PATH] == "/source/original.tif" + assert context.source_projections_by_virtual_path[PATH] is original_projection + with pytest.raises(TypeError): + context.source_projections_by_virtual_path[PATH] = original_projection + + +def test_warmed_cache_transport_reconstructs_all_derived_defaults(): + cache = RuntimeSourceBindingContextCache() + parser = CountingParser() + source_projection = projection({ORIGINAL_SOURCE_METADATA_FIELD: {"Plate": "A"}}) + context = snapshot(cache, source_projection, parser) + cache.normalized_source_metadata(source_projection.source_metadata_by_path) + cache.runtime_context_by_request_identity[(1, (), "disk", None)] = ( + SourceBindingRuntimeContext() + ) + assert cache.source_resolution_snapshots + assert cache.source_metadata_by_mapping_identity + restored = pickle.loads(pickle.dumps(cache)) + assert restored == RuntimeSourceBindingContextCache() + rebuilt = snapshot(restored, source_projection, parser) + assert rebuilt is not context + assert rebuilt.metadata_for_path(PATH) == context.metadata_for_path(PATH) From e98da6b9fa5e1f811c9e69924a1511b2f2b8b551 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:11:20 -0400 Subject: [PATCH 006/125] Exercise source snapshot transport through processing context --- ...test_function_runtime_source_projection.py | 66 +++++++++++++------ ...test_runtime_source_resolution_snapshot.py | 22 ++++++- 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index 016ae53eaf..fc4fc52482 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -3733,7 +3733,9 @@ def test_producer_anchored_pipeline_start_paths_use_exact_source_projection_bund """Producer bookkeeping must not override exact workspace source ownership.""" from openhcs.core.steps import function_runtime - from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache + from openhcs.core.runtime_source_binding_cache import ( + RuntimeSourceBindingContextCache, + ) virtual_paths = ( "A01_s001_w1_z001_t001.tif", @@ -5342,8 +5344,11 @@ def save_batch( def qualified_producer_manifest(tmp_path): parser = SourceSchemaFilenameParser() producer = SimpleNamespace( - step_scope_id="producer", step_name="Producer", pipeline_position=0, - axis_id="A01", output_dir=tmp_path, + step_scope_id="producer", + step_name="Producer", + pipeline_position=0, + axis_id="A01", + output_dir=tmp_path, ) consumer = SimpleNamespace( axis_id="A01", @@ -5357,14 +5362,22 @@ def qualified_producer_manifest(tmp_path): parsed = parser.parse_filename(f"A01_s001_w2_z{plane:03d}_t001.tif") identity = FunctionOutputIdentity( component_values=dict(parsed.component_wire_mapping()), - extension=".tif", source="test", filename_qualifier=f"Output{plane}", + extension=".tif", + source="test", + filename_qualifier=f"Output{plane}", + ) + records.append( + ProducedOutputSemantics.from_output( + producer, + tmp_path + / FunctionOutputPathAuthority.filename_for_identity(parser, identity), + identity, + output_context=AlignedImageSliceContext.main_flow( + output_key=f"Output{plane}", + artifact_kind=ImageArtifactType.value, + ), + ) ) - records.append(ProducedOutputSemantics.from_output( - producer, tmp_path / FunctionOutputPathAuthority.filename_for_identity(parser, identity), - identity, output_context=AlignedImageSliceContext.main_flow( - output_key=f"Output{plane}", artifact_kind=ImageArtifactType.value, - ), - )) store = StepOutputManifestStore() store.begin_step(producer) store.record_outputs(producer, records) @@ -5372,7 +5385,8 @@ def qualified_producer_manifest(tmp_path): def test_step_output_manifest_batch_lookup_preserves_aliases_order_and_duplicates( - qualified_producer_manifest, monkeypatch, + qualified_producer_manifest, + monkeypatch, ): store, _producer, consumer, records, parser = qualified_producer_manifest calls = [] @@ -5382,7 +5396,9 @@ def count_filename(parser, identity): calls.append(identity) return original(parser, identity) - monkeypatch.setattr(FunctionOutputPathAuthority, "filename_for_identity", count_filename) + monkeypatch.setattr( + FunctionOutputPathAuthority, "filename_for_identity", count_filename + ) paths = ( records[1].output_path, "A01_s001_w2_z001_t001.tif", @@ -5391,7 +5407,10 @@ def count_filename(parser, identity): ) result = store.producer_output_contexts_for_paths(consumer, paths, parser) assert tuple(context.output_key for context in result) == ( - "Output2", "Output1", "Output2", "Output1", + "Output2", + "Output1", + "Output2", + "Output1", ) assert len(calls) == len(records) @@ -5409,12 +5428,17 @@ def test_step_output_manifest_batch_lookup_template_deduplicates_record_aliases( assert index.matching_records(path) == (records[0],) -@pytest.mark.parametrize("path, count", [ - ("missing.tif", 0), - ("A01_s001_w2_z{plane}_t001.tif", 2), -]) +@pytest.mark.parametrize( + "path, count", + [ + ("missing.tif", 0), + ("A01_s001_w2_z{plane}_t001.tif", 2), + ], +) def test_step_output_manifest_batch_lookup_rejects_missing_and_ambiguous_templates( - qualified_producer_manifest, path, count, + qualified_producer_manifest, + path, + count, ): store, _producer, consumer, _records, parser = qualified_producer_manifest with pytest.raises(NoStepOutputManifestMatch, match=f"found {count}"): @@ -5427,10 +5451,12 @@ def test_step_output_manifest_batch_lookup_rejects_shared_basename( store, producer, consumer, records, parser = qualified_producer_manifest alias = records[0].relative_output_path other = ProducedOutputSemantics.from_output( - producer, producer.output_dir / "another" / alias, + producer, + producer.output_dir / "another" / alias, FunctionOutputIdentity( component_values=records[1].component_values, - extension=".tif", source="test", + extension=".tif", + source="test", ), output_context=records[1].output_context, ) diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py index 9033f38827..d934eee0b1 100644 --- a/tests/unit/test_runtime_source_resolution_snapshot.py +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -7,6 +7,7 @@ from polystore.virtual_workspace import SourcePixelRef from openhcs.constants.constants import AllComponents +from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache from openhcs.core.source_binding_selection import ( DeclaredSourceMetadataRecord, @@ -258,8 +259,14 @@ def test_snapshot_owns_position_and_projection_map_views(): context.source_projections_by_virtual_path[PATH] = original_projection -def test_warmed_cache_transport_reconstructs_all_derived_defaults(): - cache = RuntimeSourceBindingContextCache() +@pytest.mark.parametrize("context_owned", (False, True)) +def test_warmed_cache_transport_reconstructs_all_derived_defaults(context_owned): + processing_context = ProcessingContext(axis_id="A01") + cache = ( + processing_context.runtime_source_binding_context_cache + if context_owned + else RuntimeSourceBindingContextCache() + ) parser = CountingParser() source_projection = projection({ORIGINAL_SOURCE_METADATA_FIELD: {"Plate": "A"}}) context = snapshot(cache, source_projection, parser) @@ -269,7 +276,16 @@ def test_warmed_cache_transport_reconstructs_all_derived_defaults(): ) assert cache.source_resolution_snapshots assert cache.source_metadata_by_mapping_identity - restored = pickle.loads(pickle.dumps(cache)) + restored_owner = pickle.loads( + pickle.dumps(processing_context if context_owned else cache) + ) + restored = ( + restored_owner.runtime_source_binding_context_cache + if context_owned + else restored_owner + ) + if context_owned: + assert restored_owner.axis_id == "A01" assert restored == RuntimeSourceBindingContextCache() rebuilt = snapshot(restored, source_projection, parser) assert rebuilt is not context From 0c9521abb9b4bb1a3ef2a2cb86003a48d37fffd6 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:12:33 -0400 Subject: [PATCH 007/125] Preserve original metadata record hash projection --- openhcs/core/source_binding_selection.py | 2 +- tests/unit/test_runtime_source_resolution_snapshot.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openhcs/core/source_binding_selection.py b/openhcs/core/source_binding_selection.py index 13e7673ba9..c4c9ca4007 100644 --- a/openhcs/core/source_binding_selection.py +++ b/openhcs/core/source_binding_selection.py @@ -150,7 +150,7 @@ def __eq__(self, other: object) -> bool: return self.fields == other.fields def __hash__(self) -> int: - return hash(self.fields) + return hash((self.fields,)) @dataclass(frozen=True, slots=True, eq=False) diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py index d934eee0b1..e49a9c0f8a 100644 --- a/tests/unit/test_runtime_source_resolution_snapshot.py +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -224,6 +224,7 @@ def test_declared_and_resolved_records_preserve_family_value_identity(): ) assert declared == resolved assert hash(declared) == hash(resolved) + assert hash(declared) == hash((declared.fields,)) assert declared != SourceMetadataRecord.from_mapping({"site": 2, "well": "A01"}) nested = {ORIGINAL_SOURCE_METADATA_FIELD: {"Plate": "A"}} with pytest.raises(TypeError): From bc2beb50152c5db0d42d976d5a6ab16aecf3c26f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:16:47 -0400 Subject: [PATCH 008/125] Record materialization source ratchet qualification (#384) --- .../validation/materialization_plumbing_384_20261001.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/validation/materialization_plumbing_384_20261001.rst b/docs/validation/materialization_plumbing_384_20261001.rst index a0dc46bcbf..4d7669a4ea 100644 --- a/docs/validation/materialization_plumbing_384_20261001.rst +++ b/docs/validation/materialization_plumbing_384_20261001.rst @@ -85,6 +85,13 @@ and export, all three runtime observation modes, pickle transport after resource release, exact combined worker/parent export paths, materialization formats, metadata publication/reconciliation, runtime/viewer validation and ZMQ consumers. -R0/R1 source guard qualification, representative ordinary unprofiled timing and +The original unmodified R0 guard passes for a264ee95546ab17a3ded40f81759cc6234d08817 +against main e341ac9b55cb2b6a143df9083785b296c23000df. There are no positive debt +deltas; ZMQExecutionServer god-class excess decreases by eight lines and the +artifact materialization foreign-absence probe decreases by one. This revision +includes the normal merge of main's fixture/documentation fix; production sources +are identical to the 267-test candidate. + +Combined R1 source qualification, representative ordinary unprofiled timing and native numerical parity remain required before promotion. No benchmark ratios or speedup claim are attached to this receipt. From ea73dadc1dbdf518c267da6545cd012528bebaef Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:28:35 -0400 Subject: [PATCH 009/125] Enforce resolved metadata invariants and verify adapter provenance --- openhcs/core/runtime_source_binding_cache.py | 2 +- openhcs/core/source_binding_selection.py | 19 ++++--- .../test_runtime_adapter_source_artifacts.py | 50 ++++++++++++++++--- ...test_runtime_source_resolution_snapshot.py | 11 ++++ 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/openhcs/core/runtime_source_binding_cache.py b/openhcs/core/runtime_source_binding_cache.py index d067f2b5fe..0231210c85 100644 --- a/openhcs/core/runtime_source_binding_cache.py +++ b/openhcs/core/runtime_source_binding_cache.py @@ -125,7 +125,7 @@ def source_pattern_context( self.source_resolution_snapshots[key] = snapshot return snapshot.context - def __reduce__(self) -> tuple[type["RuntimeSourceBindingContextCache"], tuple[()]]: + def __reduce__(self) -> tuple[type[RuntimeSourceBindingContextCache], tuple[()]]: """Transport reconstructs all derived caches from declaration defaults.""" return type(self), () diff --git a/openhcs/core/source_binding_selection.py b/openhcs/core/source_binding_selection.py index c4c9ca4007..fde043f4dc 100644 --- a/openhcs/core/source_binding_selection.py +++ b/openhcs/core/source_binding_selection.py @@ -170,20 +170,25 @@ def resolve( class ResolvedSourceMetadataRecord(SourceMetadataRecord): """Immutable path-specific metadata resolved at a runtime snapshot boundary.""" - @classmethod - def from_resolved_mapping( - cls, metadata: SourceMetadataMapping - ) -> "ResolvedSourceMetadataRecord": - return cls( + def __post_init__(self) -> None: + object.__setattr__( + self, + "fields", tuple( ( str(key), SourceBindingRuntimeMetadataNormalizer.normalized_value(value), ) - for key, value in metadata.items() - ) + for key, value in self.fields + ), ) + @classmethod + def from_resolved_mapping( + cls, metadata: SourceMetadataMapping + ) -> "ResolvedSourceMetadataRecord": + return cls(tuple(metadata.items())) + def resolve( self, path: str, diff --git a/tests/unit/test_runtime_adapter_source_artifacts.py b/tests/unit/test_runtime_adapter_source_artifacts.py index 28595b345a..dd4e4357fe 100644 --- a/tests/unit/test_runtime_adapter_source_artifacts.py +++ b/tests/unit/test_runtime_adapter_source_artifacts.py @@ -2,6 +2,7 @@ from types import SimpleNamespace import numpy as np +import pytest from polystore.base import ensure_storage_registry, storage_registry from polystore.filemanager import FileManager from scipy.io import savemat @@ -12,7 +13,9 @@ from openhcs.core.component_group_scope import RuntimeExecutionAxisScope from openhcs.core.runtime_adapters import RuntimeAdapterRequest from openhcs.core.runtime_image_loading import ImagePayloadSourceMetadataContext -from openhcs.core.runtime_image_values import image_payload_data +from openhcs.core.runtime_image_values import image_payload_data, image_payload_metadata +from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache +from openhcs.core.source_binding_selection import SourcePatternResolutionContext from openhcs.core.runtime_object_labels import ObjectLabelSet from openhcs.core.source_binding_workspace import SourceBindingWorkspaceProjector from openhcs.core.source_bindings import ( @@ -54,6 +57,7 @@ def _filemanager() -> FileManager: def test_source_artifact_inputs_share_workspace_vfs_and_contract_resolution( tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: source_root = tmp_path / "source" workspace_root = tmp_path / "workspace" @@ -141,6 +145,7 @@ def binding( filemanager=filemanager, microscope_handler=microscope_handler, runtime_source_workspace_projection_cache=projection_cache, + runtime_source_binding_context_cache=RuntimeSourceBindingContextCache(), source_image_set_identity_policy=SourceImageSetIdentityPolicy( frozenset((AllComponents.CHANNEL,)) ), @@ -215,12 +220,11 @@ def request() -> RuntimeAdapterRequest: ) illumination_request = request() + illumination_payload = illumination_request.source_artifact_payload( + illumination_binding.input_spec().ref() + ) np.testing.assert_array_equal( - image_payload_data( - illumination_request.source_artifact_payload( - illumination_binding.input_spec().ref() - ) - ), + image_payload_data(illumination_payload), illumination[np.newaxis, ...], ) @@ -245,3 +249,37 @@ def request() -> RuntimeAdapterRequest: ) assert isinstance(label_set, ObjectLabelSet) np.testing.assert_array_equal(label_set.labels, labels[np.newaxis, ...]) + + # Exercise the actual source adapter and VFS under the independent live + # selector route. Payload identity belongs to the unchanged typed source + # projection, not the selector's resolved metadata record representation. + with monkeypatch.context() as live_selector: + live_selector.setattr( + RuntimeSourceBindingContextCache, + "source_pattern_context", + lambda self, **kwargs: SourcePatternResolutionContext.from_projection( + **kwargs + ), + ) + live_illumination = request().source_artifact_payload( + illumination_binding.input_spec().ref() + ) + live_labels = request().source_artifact_payload(labels_spec.ref()) + for produced, live in ( + (illumination_payload, live_illumination), + (labels_payload, live_labels), + ): + np.testing.assert_array_equal( + image_payload_data(produced), image_payload_data(live) + ) + produced_metadata = image_payload_metadata(produced) + live_metadata = image_payload_metadata(live) + assert produced_metadata == live_metadata + assert ( + produced_metadata.source_provenance.source_identity.identity + == live_metadata.source_provenance.source_identity.identity + ) + assert ( + produced_metadata.source_provenance.represented_source_identities + == live_metadata.source_provenance.represented_source_identities + ) diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py index e49a9c0f8a..1a2662a51c 100644 --- a/tests/unit/test_runtime_source_resolution_snapshot.py +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -260,6 +260,17 @@ def test_snapshot_owns_position_and_projection_map_views(): context.source_projections_by_virtual_path[PATH] = original_projection +def test_resolved_direct_constructor_owns_deep_immutable_invariant(): + nested = {"Plate": "before"} + resolved = ResolvedSourceMetadataRecord(((ORIGINAL_SOURCE_METADATA_FIELD, nested),)) + nested["Plate"] = "after" + assert resolved[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "before" + with pytest.raises(TypeError): + resolved[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] = "invalid" + with pytest.raises(TypeError, match="Source metadata scalar values"): + ResolvedSourceMetadataRecord((("nested", {"unsupported": {"deep": 1}}),)) + + @pytest.mark.parametrize("context_owned", (False, True)) def test_warmed_cache_transport_reconstructs_all_derived_defaults(context_owned): processing_context = ProcessingContext(axis_id="A01") From 74ecf9d3ea167a24cc316be11e1a71f43795de7a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 14:41:21 -0400 Subject: [PATCH 010/125] Document integrated runtime plumbing qualification and limits (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/validation/shared_runtime_plumbing_384_20261001.rst diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst new file mode 100644 index 0000000000..b7fee9f740 --- /dev/null +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -0,0 +1,64 @@ +Shared runtime plumbing qualification +==================================== + +Issue #384 covers repeated source resolution, metadata construction and artifact +rendering outside callable execution. This branch combines three production +dependencies under their existing lifecycle owners. It does not yet establish an +end-to-end speedup. + +The unmodified-main light trace located about 3.44 seconds in loading, unstacking, +saving and finalization for the representative 3D pipeline. CP request and output +context construction account for another approximately 1.216 seconds. These are +diagnostic bounds, not additive nested timer totals or accepted benchmark clocks. +Repeated artifact rendering alone is bounded at about 0.297 seconds; saved plane +projection replay estimates about 0.20 seconds for that individual component. +Neither is admitted as a sufficient standalone performance route. The integrated +boundary must demonstrate a material improvement in ordinary execution. + +Production authorities +---------------------- + +``ImageMetadataProjection`` owns the common single-pass image metadata algorithm. +Source-plane and leading-axis specializations provide their declared projection +semantics through inheritance. Provenance constructors consume the actual +``SourceImageProvenance`` constructor defaults, avoiding absent-field allocation +without a second default roster. Mutable scalar payload metadata retains its +independent ownership and mutation controls. + +``SourceMetadataRecord`` owns declared/parser/rule resolution. Its declared leaf +retains live input semantics, while its resolved leaf enforces owned immutable +metadata at construction. ``RuntimeSourceResolutionSnapshot`` belongs to the +existing context-local runtime source cache and retains its determining inputs. +Only the explicit execution factories take this boundary; direct constructors and +unknown paths preserve live resolution. Derived caches are discarded on process +transport. The original adapter provenance construction remains authoritative: +deep-freezing selector records must not change repr-based image identities. + +``MaterializationBatch`` owns rendering and saving. Actual saved outputs flow into +the existing metadata target family, step observation, worker observation and +plate observation. Export paths derive from these outcomes after payload resource +release. Removed rerender helpers and the old context-derived export path API have +no remaining consumers. Historical debug reuse retains explicitly named previews; +these do not claim new successful saves. The component receipt describes the +retained CSV consolidation and independent viewer expectation consumers. + +Qualification and disposition +---------------------------- + +The integrated source at b88eb36604f22e12f3fbc4123c251b0f12b8e48f passed 1,362 +behavioral tests in 43.25 seconds. After the normal merge of main's classification +fix 7a5ee630c, all 18 classification controls passed in 3.35 seconds. Each of the +three component source changes passed the original unmodified native R0 guard. +Combined R0/R1, current-main correctness synchronization, ordinary paired pipeline +timings and CellProfiler numerical parity remain required before promotion. + +The earlier runtime-plumbing prototype 2253ec63b is rejected. Its paired timing +did not establish a gain and its original R0 gate failed. It is not part of this +branch or the accepted evidence. Failed controls and diagnostic observations are +retained in the external benchmark evidence directory rather than replaced with +the fastest observations. + +Pipeline clocks exclude ZMQ server startup, mandatory registry/kernel prewarming +and shutdown. Timed runs use the ordinary public OUTCOMES path and default memory +observer, without profiling injection. Candidate and baseline must have frozen +clean source revisions and the same dependencies, affinity and environment. From 97f1160aade32613db4fe544339a8457dfde3fa3 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 15:15:11 -0400 Subject: [PATCH 011/125] fix: retain raster admission for actual artifact publication --- .../materialization_plumbing_384_20261001.rst | 35 +++++ openhcs/core/steps/function_outputs.py | 1 + tests/unit/test_function_outputs.py | 124 ++++++++++++++++++ 3 files changed, 160 insertions(+) diff --git a/docs/validation/materialization_plumbing_384_20261001.rst b/docs/validation/materialization_plumbing_384_20261001.rst index 4d7669a4ea..8125d2b4b7 100644 --- a/docs/validation/materialization_plumbing_384_20261001.rst +++ b/docs/validation/materialization_plumbing_384_20261001.rst @@ -95,3 +95,38 @@ are identical to the 267-test candidate. Combined R1 source qualification, representative ordinary unprofiled timing and native numerical parity remain required before promotion. No benchmark ratios or speedup claim are attached to this receipt. + +Actual array outputs and raster publication +------------------------------------------ + +The first combined native comparison retained 28 numerically equivalent cases +and two illumination metadata failures. The saved illumination NumPy arrays were +actual outputs, but their NumPy-only directory had incorrectly become eligible +for raster image inventory publication. The existing FileManager inventory +includes declared raster formats; NumPy array exports are outside that inventory. +The candidate had removed OutputTarget's inherited ``contains_images`` admission +gate while switching directory discovery to actual saved outputs. + +The correction restores that existing nominal gate. Actual NumPy saved outputs +and their observation locations remain intact; publication and completed-plate +reconciliation consume only directories with images in the existing inventory. +No second format registry, rerendering fallback or empty source projection was +introduced. + +Independent controls use a real FileManager and physical NumPy-only and mixed +TIFF/NumPy outputs. The original candidate fails the NumPy-only control with the +same empty SourceProjectionSet error; its mixed-directory control passes. Both +controls pass after restoration, verifying exact saved pixels, actual observation +paths, raster-only inventory and typed calibration. The focused red and green +receipts are ``materialization-npy-publication-red-v3-20261001.log/xml`` and +``materialization-npy-publication-green-20261001.log/xml`` in the external benchmark +evidence directory. Earlier import-order and typed-unit fixture errors are also +retained separately; they are harness failures, not the product regression. + +The expanded eleven-module consumer gate passes all 269 tests using only the +original nominal viewer route fixture from the root test configuration. Two +initial broader invocations retained 264 passes and five missing-fixture setup +errors; the corrected fixture registration preserves those same five controls. +The final receipt is ``materialization-npy-publication-consumers-v3-20261001.log/xml``. +Full native parity and fresh accepted timing remain required for the corrected +combined source; the original 28-pass/two-error receipt is not replaced. diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py index eda426ccc8..05b7fb539c 100644 --- a/openhcs/core/steps/function_outputs.py +++ b/openhcs/core/steps/function_outputs.py @@ -1164,6 +1164,7 @@ def production_targets( target for directory in directories for target in (self.for_directory(directory),) + if target.contains_images(context) ) def reconciliation_targets( diff --git a/tests/unit/test_function_outputs.py b/tests/unit/test_function_outputs.py index a97b9bbb33..7d887b0825 100644 --- a/tests/unit/test_function_outputs.py +++ b/tests/unit/test_function_outputs.py @@ -54,6 +54,7 @@ SOURCE_VOXEL_SPACING_FIELD, SOURCE_VOXEL_SPACING_UNIT_FIELD, SourceVoxelSpacing, + SourceVoxelSpacingUnit, ) from openhcs.core.source_projection import SourceProjectionMetadataSerializer from openhcs.core.source_spatial_domain import SourceSpatialDomain @@ -2388,3 +2389,126 @@ def test_declared_image_destinations_publish_and_reconcile_after_value_cleanup( assert projection.component_value(AllComponents.CHANNEL) == "2" assert projection.source_metadata["site"] == "1" np.testing.assert_array_equal(tifffile.imread(tmp_path / path), pixels) + + +@pytest.mark.parametrize("with_raster", [False, True]) +def test_actual_array_exports_only_publish_declared_raster_inventory( + tmp_path, with_raster +): + """A saved NumPy array is an actual output, without being a raster inventory.""" + filemanager = FileManager( + { + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + } + ) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + context.runtime_value_store = RuntimeValueStore() + context.metadata_cache = {} + context.tiff_config = None + plan = function_step_plan("Independent array and raster exports") + plan.streaming_configs = {} + plan.write_backend = Backend.MEMORY.value + plan.output_plate_root = str(tmp_path) + plan.output_dir = tmp_path / "images" + plan.sub_dir = "images" + plan.analysis_results_dir = str(tmp_path / "results") + plan.create_openhcs_metadata = True + plan.runtime_artifact_materialization = RuntimeArtifactMaterializationPlan( + persistent_enabled=True, + persistent_backend=Backend.DISK.value, + ) + context.step_plans = {plan.step_index: plan} + pixels = np.arange(20, dtype=np.float32).reshape(4, 5) / 7 + payload = ImageMetadataPayload( + pixels, + ImagePayloadMetadata( + source_component_metadata={ + "well": "A01", + "site": "1", + "channel": "2", + "z_index": "1", + "timepoint": "1", + }, + source_dtype="float32", + source_voxel_spacing=SourceVoxelSpacing( + (0.65, 0.65), SourceVoxelSpacingUnit.MICROMETERS + ), + ), + ) + declarations = [("NumericField", "exports/Illum.npy")] + if with_raster: + declarations.append(("RasterField", "exports/A01_s001_w2_z001_t001.tif")) + for name, relative_path in declarations: + output_plan = ArtifactOutputPlan( + name=name, + path=f"/memory/{name}.pkl", + artifact_type=ImageArtifactType, + materialization=MaterializationSpec( + ImageFileOptions( + filename_suffix=Path(relative_path).suffix, + relative_path_template=relative_path, + ) + ), + ) + plan.artifact_outputs[output_plan.ref()] = output_plan + context.runtime_value_store.record( + RuntimeValue.normalize(output_plan, payload, axis_id="A01"), + path=output_plan.path, + backend=Backend.MEMORY.value, + ) + + materializations = RuntimeArtifactMaterializationAuthority.materialize( + context, plan + ) + saved_outputs = tuple( + output + for artifact in materializations + for output in artifact.outputs_for_backend(Backend.DISK.value) + ) + expected_paths = { + tmp_path / "images" / relative_path for _name, relative_path in declarations + } + assert {Path(output.path) for output in saved_outputs} == expected_paths + assert all(path.is_file() for path in expected_paths) + np.testing.assert_array_equal( + np.load(tmp_path / "images/exports/Illum.npy", allow_pickle=False), pixels + ) + observed_locations = tuple( + location + for artifact in materializations + for locations in ( + artifact.observation(plan).materialized_locations_by_address.values() + ) + for location in locations + ) + assert {Path(location.path) for location in observed_locations} == expected_paths + + OpenHCSMetadataWriter.write( + context, plan, artifact_materializations=materializations + ) + context.runtime_value_store.clear() + OpenHCSMetadataWriter.finalize_completed_plate({"A01": context}) + metadata_path = tmp_path / "openhcs_metadata.json" + if not with_raster: + assert not metadata_path.exists() + return + np.testing.assert_array_equal( + tifffile.imread(tmp_path / "images/exports/A01_s001_w2_z001_t001.tif"), pixels + ) + subdirectories = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES] + assert set(subdirectories) == {"images/exports"} + subdirectory = subdirectories["images/exports"] + assert subdirectory[FIELDS.IMAGE_FILES] == [ + "images/exports/A01_s001_w2_z001_t001.tif" + ] + entries = VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries + raster = entries["images/exports/A01_s001_w2_z001_t001.tif"] + assert raster.source_alias == "RasterField" + assert raster.image_metadata.source_voxel_spacing.values_zyx == (0.65, 0.65) + assert ( + raster.image_metadata.source_voxel_spacing.native_coordinate_unit + == "micrometer" + ) From 6ee21ac8cbac4763619166c06fc6d7fb6da151db Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 16:01:03 -0400 Subject: [PATCH 012/125] docs: record repaired qualification and formal issue-link audit for #384 --- .../shared_runtime_plumbing_384_20261001.rst | 108 +++++++++++++++++- 1 file changed, 102 insertions(+), 6 deletions(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index b7fee9f740..9f09bf5cae 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -45,12 +45,108 @@ retained CSV consolidation and independent viewer expectation consumers. Qualification and disposition ---------------------------- -The integrated source at b88eb36604f22e12f3fbc4123c251b0f12b8e48f passed 1,362 -behavioral tests in 43.25 seconds. After the normal merge of main's classification -fix 7a5ee630c, all 18 classification controls passed in 3.35 seconds. Each of the -three component source changes passed the original unmodified native R0 guard. -Combined R0/R1, current-main correctness synchronization, ordinary paired pipeline -timings and CellProfiler numerical parity remain required before promotion. +The integrated source passed 1,362 behavioral tests. Each of the three components +passed the original unmodified R0 guard. Final combined R0/R1 passed against main +91dc52d4040890e1b88494acba7368196f6cd1e8 at tested source +dc1a371b06079ff249c8c6432953d21e0c05917b, without increasing debt allowances or +source checks. The main synchronization passed 85 help/output/source-resolution +controls. These source ratchets are scoped guards, not a formal proof of every +dynamic binding. + +The first complete saved-native gate exposed two illumination-correction +publication failures: actual numeric NPY saves admitted directories without any +raster images into metadata publication. The inherited raster-inventory admission +now governs directories derived from successful saves. A real FileManager NPY-only +control reproduced the failure; a mixed TIFF/NPY control preserves exact pixels, +raster inventory, alias and calibration. All 269 related controls passed. The +repaired full gate completed: all 30 cases equivalent, zero differences. The +original 28-pass/two-error gate is retained. + +Repaired ordinary ABBA evidence +------------------------------ + +Two main and two candidate observations per case were retained, without excluding +samples. Revisions are the exact dc1a371b/91dc52d4 sources above. In seconds: + +========================= ======================= ======================= +Case Mean execution Mean pipeline total +========================= ======================= ======================= +3D monolayer 10.278553 -> 9.657654 12.241441 -> 11.607624 +Imaging Flow Cytometry 15.506204 -> 15.423871 17.719883 -> 17.949389 +Advanced segmentation 8.658128 -> 8.887496 11.158074 -> 11.397346 +========================= ======================= ======================= + +3D execution improves 6.04%, while advanced execution regresses 2.65%. The default +memory observer's mean peak process-tree RSS changes by +14.65% for 3D, -3.41% for +Imaging Flow Cytometry and +10.87% for advanced segmentation. Process-tree RSS can +count shared fork pages repeatedly; these figures do not prove distinct physical +allocation. Candidate memory varies substantially between the two observations. +This is mixed evidence, not an accepted broad optimization. PR394 remains draft. + +The pre-publication-repair eight-sweep observations remain retained separately; +their larger improvement is not substituted for the repaired source's results. +External evidence: shared-runtime-plumbing-repaired-qualified-abba-summary- +20261001.json and every corresponding source/input/observation receipt in the +openhcs-benchmark-runs evidence directory. + +Fresh native qualification and next dominant route +------------------------------------------------ + +The revision-pinned matched Imaging Flow Cytometry driver completed three observed +native CP/OpenHCS repetitions and one warmup. All observed repetitions have zero +output differences under the existing numerical tolerances, exact discrete/schema +checks and strict declared output inventory. The input is one well with two native +image sets, one OpenHCS worker and one native job; the configured worker start +method is fork. Native and candidate phase boundaries are retained separately. +The driver's timing_claim explicitly remains none until first-module/first-axis +boundary equivalence is qualified. No native throughput ratio is asserted here; +the physical 3D volume/plane consumer gap also remains unfinished. + +Fresh paired diagnostic traces cover all three workloads with identical owner +hooks. Their high-frequency instrumentation perturbs costs, so these are authority +and call-count evidence, not accepted timing clocks. The 3D trace reduces +SourceImageIdentity construction from 142,412 to 78,769, but still calls +SourceMetadataRoleView.scalar_items 194,349 times and source metadata field identity +1,120,064 times. Image-processing invocation remains approximately unchanged in +these traces. Repeated metadata queries are the next potentially material generic +route; saved representative replay must establish its transferable payoff before +another production authority is admitted. Immutable query ownership must retain +live public mapping behavior, lazy validation, alias/role semantics and image +identity fingerprints. + +Source snapshots structurally retain historical projection owners, but output +publication does not normally replace these workspaces' cached metadata documents. +The observed RSS increase is therefore not attributed to snapshot generations. +Rendered outputs also remain visible through immediate publication; many arrays +already belong to runtime storage. Per-process memory and ownership diagnostics +must distinguish extra references from newly allocated buffers before changing +that boundary. + +The subsequent normal main396 merge changes documentation, knowledge resources +and five knowledge-transfer test assertions. OpenHCS production source bytes are +unchanged from the corresponding measured candidate and baseline revisions; the +existing source/benchmark evidence remains pinned to its original revisions. + +Issue and branch audit +---------------------- + +The source components are integrated in actual draft PR394, formally linked to +issue384. A branch/history audit covered 46 local worktrees and 194 remote branches; +the two subsequently created branches have PR396 (merged) and PR397 (draft). +Current branch tips were checked beyond historical PR association, including exact +source patch equivalence for squash/rebase history. No additional useful +unpublished fix branch was identified. Rejected prototypes and obsolete planning +branches are not nominated as unfinished production fixes. + +The fresh open-issue audit verifies 7 of 29 have formal closing PR references; +22 still have no actual future fixing PR. Mere cross-references and diagnosis-only +drafts are not counted. Missing manual closing links were added for +138/207, 213/217, 320/217, 355/358 and 395/397. Existing 384/394 and 386/388 links +have closing cross-reference events. GitHub's official manual mutation is a no-op +for these already linked pairs; no exact ConnectedEvent is claimed for them. +Audit evidence: current-open-issues-connected-closing-audit-20261001.json/.md, +remaining-remote-branch-pr-audit-20261001.json and +local-worktree-branch-inventory-20261001.json in the external evidence directory. The earlier runtime-plumbing prototype 2253ec63b is rejected. Its paired timing did not establish a gain and its original R0 gate failed. It is not part of this From 5d1f1a781eda851b4783b68db342b76eeef9dea9 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 17:28:43 -0400 Subject: [PATCH 013/125] refactor(runtime): store cached image payloads directly (#384) --- openhcs/core/runtime_stack_cache.py | 13 +++---------- openhcs/core/steps/function_runtime.py | 2 +- tests/unit/test_runtime_stack_cache_lifetime.py | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/openhcs/core/runtime_stack_cache.py b/openhcs/core/runtime_stack_cache.py index 4b92cb0020..6c2248777b 100644 --- a/openhcs/core/runtime_stack_cache.py +++ b/openhcs/core/runtime_stack_cache.py @@ -27,18 +27,11 @@ def from_paths( return cls(tuple(str(path) for path in paths), str(memory_type)) -@dataclass(frozen=True, slots=True) -class RuntimeImageStackCacheValue: - """Cached payload for an ordered runtime-slice path set.""" - - stack: RuntimeArrayData - - @dataclass(slots=True) class RuntimeImageStackCache: """Processing-context-local cache for adjacent step stack reuse.""" - stacks: dict[RuntimeImageStackCacheKey, RuntimeImageStackCacheValue] = field( + stacks: dict[RuntimeImageStackCacheKey, RuntimeArrayData] = field( default_factory=dict ) @@ -47,7 +40,7 @@ def get( paths: tuple[str, ...], *, memory_type: str, - ) -> RuntimeImageStackCacheValue | None: + ) -> RuntimeArrayData | None: """Return a cached stack for the exact ordered path and memory contract.""" return self.stacks.get( RuntimeImageStackCacheKey.from_paths(paths, memory_type=memory_type) @@ -63,7 +56,7 @@ def store( """Store a stack produced for a just-saved ordered path set.""" self.stacks[ RuntimeImageStackCacheKey.from_paths(paths, memory_type=memory_type) - ] = RuntimeImageStackCacheValue(stack=stack) + ] = stack def discard_paths(self, paths: tuple[str, ...]) -> None: """Discard cached stacks that include any of the supplied paths.""" diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index 266ed9b128..8b4ad2f902 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -2821,7 +2821,7 @@ def _load_input_stack(self) -> PatternGroupData: metadata_mode=metadata_mode, ).compose() else: - main_data_stack = cached_stack.stack + main_data_stack = cached_stack return PatternGroupData( matching_files=matching_files, diff --git a/tests/unit/test_runtime_stack_cache_lifetime.py b/tests/unit/test_runtime_stack_cache_lifetime.py index 78470ea371..f735926289 100644 --- a/tests/unit/test_runtime_stack_cache_lifetime.py +++ b/tests/unit/test_runtime_stack_cache_lifetime.py @@ -38,7 +38,7 @@ def test_release_drops_all_cached_views_and_allows_reuse(context_owned: bool) -> assert all(cache.get(entry, memory_type="numpy") is None for entry in paths) replacement = np.zeros((1, 8, 8), dtype=np.uint16) cache.store(paths[0], memory_type="numpy", stack=replacement) - assert cache.get(paths[0], memory_type="numpy").stack is replacement + assert cache.get(paths[0], memory_type="numpy") is replacement def test_context_image_cache_release_preserves_runtime_artifacts_and_observations() -> ( From b4cbcb6cb92b6181d822f59188dcb46f89a41aa8 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 17:38:13 -0400 Subject: [PATCH 014/125] perf(metadata): derive geometry from transaction-owned projections (#384) --- openhcs/core/virtual_workspace_metadata.py | 36 ++++-- tests/unit/test_persisted_image_metadata.py | 124 +++++++++++++++++++- 2 files changed, 148 insertions(+), 12 deletions(-) diff --git a/openhcs/core/virtual_workspace_metadata.py b/openhcs/core/virtual_workspace_metadata.py index 6ef9709f73..ff4abc6d0c 100644 --- a/openhcs/core/virtual_workspace_metadata.py +++ b/openhcs/core/virtual_workspace_metadata.py @@ -5,7 +5,7 @@ import json import logging import os -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType @@ -97,7 +97,12 @@ def update(data: dict[str, Any] | None) -> dict[str, Any]: } else: subdirectory[key] = value - self._update_projection_geometry(subdirectory) + self._update_projection_geometry( + subdirectory, + VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries.values(), + ) return data self._execute_update( @@ -139,7 +144,12 @@ def update(data): subdirectory_name, {} ) self._merge_source_projection_fields(subdirectory, projection_metadata) - self._update_projection_geometry(subdirectory) + self._update_projection_geometry( + subdirectory, + VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries.values(), + ) return data self._execute_update(metadata_path, update) @@ -238,21 +248,25 @@ def update(data): subdirectory[serializer.MAIN_FIELD] = True if results_dir is not None: subdirectory[serializer.RESULTS_DIR_FIELD] = results_dir - self._update_projection_geometry(subdirectory) + self._update_projection_geometry( + subdirectory, + (projection for projection, _path in retained_paths), + ) return data self._execute_update(metadata_path, update) @staticmethod - def _update_projection_geometry(subdirectory: dict[str, Any]) -> None: - entries = VirtualWorkspaceSourceProjectionEntries.from_subdirectory( - subdirectory - ).entries - if not entries: - return + def _update_projection_geometry( + subdirectory: dict[str, Any], + source_projections: Iterable[SourceProjection], + ) -> None: + """Derive geometry from the transaction's current nominal projections.""" unique_projections: dict[tuple[object, ...], SourceProjection] = {} - for projection in entries.values(): + for projection in source_projections: unique_projections.setdefault(projection.identity_key, projection) + if not unique_projections: + return projections = SourceProjectionSet(tuple(unique_projections.values())) subdirectory[FIELDS.GRID_DIMENSIONS] = ( SourceTileLayout.metadata_grid_dimensions(projections) diff --git a/tests/unit/test_persisted_image_metadata.py b/tests/unit/test_persisted_image_metadata.py index 7df244dfc0..6dd325823b 100644 --- a/tests/unit/test_persisted_image_metadata.py +++ b/tests/unit/test_persisted_image_metadata.py @@ -33,7 +33,13 @@ SourceProjectionMetadataSerializer, ) from openhcs.core.source_spatial_domain import SourceSpatialDomain -from openhcs.core.virtual_workspace_metadata import FIELDS, AtomicMetadataWriter +from openhcs.core.virtual_workspace_metadata import ( + FIELDS, + AtomicMetadataWriter, + MetadataWriteError, + VirtualWorkspaceSourceProjectionEntries, +) +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser from openhcs.serialization.json import to_jsonable @@ -189,6 +195,122 @@ def merge(well): assert len(subdir[FIELDS.SOURCE_METADATA]) == 4 +def calibrated_projection(well, path, pixel_size): + spacing = SourceVoxelSpacing((pixel_size, pixel_size)) + source_metadata = {} + spacing.merge_into(source_metadata, path=path) + return SourcePlaneProjection( + OpenHCSPlaneAddress.from_values(well, 1, 1, 1, 1), + SourcePixelRef("disk", path), + source_metadata=source_metadata, + image_metadata=metadata_fixture().replace_fields(source_voxel_spacing=spacing), + ) + + +def publish_projection_inventory(writer, path, projection_metadata, saved_paths): + writer.publish_source_projection_metadata( + path, + "images", + projection_metadata, + serializer=SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()), + saved_image_paths=saved_paths, + microscope_handler_name="VirtualWorkspaceMicroscopeHandler", + source_filename_parser_name="SourceSchemaFilenameParser", + component_labels={}, + backend="disk", + is_main=True, + results_dir=None, + ) + + +@pytest.mark.parametrize( + "merge_method", ("merge_subdirectory_metadata", "merge_source_projection_metadata") +) +def test_atomic_projection_merge_refreshes_geometry_from_current_document( + tmp_path, merge_method +): + path = tmp_path / "openhcs_metadata.json" + writer = AtomicMetadataWriter() + serializer = SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()) + first = calibrated_projection("A01", "images/first.tif", 0.5) + second = calibrated_projection("A02", "images/second.tif", 2.0) + first_fields = serializer.projection_fields(((first, "images/first.tif"),)) + writer.replace_subdirectory_metadata(path, "images", first_fields) + second_fields = serializer.projection_fields(((second, "images/second.tif"),)) + if merge_method == "merge_subdirectory_metadata": + # This API replaces the projection field with the authoritative new list. + writer.merge_subdirectory_metadata(path, {"images": second_fields}) + expected_pixel_size = 2.0 + else: + writer.merge_source_projection_metadata(path, "images", second_fields) + expected_pixel_size = 1.0 # Conflicting calibration has no physical scalar. + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert subdir[FIELDS.PIXEL_SIZE] == expected_pixel_size + + +def test_final_publication_prunes_geometry_and_preserves_other_directories(tmp_path): + path = tmp_path / "openhcs_metadata.json" + writer = AtomicMetadataWriter() + serializer = SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()) + retained = calibrated_projection("A01", "images/retained.tif", 0.5) + deleted = calibrated_projection("A02", "images/deleted.tif", 2.0) + elsewhere = calibrated_projection("A03", "other/retained.tif", 0.5) + projection_paths = ( + (retained, "images/retained.tif"), + (deleted, "images/deleted.tif"), + (elsewhere, "other/retained.tif"), + ) + writer.merge_source_projection_metadata( + path, "images", serializer.projection_fields(projection_paths) + ) + before = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert before[FIELDS.PIXEL_SIZE] == 1.0 + publish_projection_inventory(writer, path, None, ("images/retained.tif",)) + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert subdir[FIELDS.PIXEL_SIZE] == 0.5 + assert subdir[FIELDS.IMAGE_FILES] == ["images/retained.tif"] + entries = VirtualWorkspaceSourceProjectionEntries.from_subdirectory(subdir).entries + assert set(entries) == {"images/retained.tif", "other/retained.tif"} + assert entries["images/retained.tif"].image_metadata == retained.image_metadata + assert entries["other/retained.tif"].image_metadata == elsewhere.image_metadata + # A failed final inventory cannot publish fields or prune existing records. + before_bytes = path.read_bytes() + with pytest.raises(MetadataWriteError, match="lack typed produced addresses"): + publish_projection_inventory(writer, path, None, ("images/unowned.tif",)) + assert path.read_bytes() == before_bytes + + +def test_concurrent_step_publication_retains_records_outside_partial_inventory( + tmp_path, +): + path = tmp_path / "openhcs_metadata.json" + writer = AtomicMetadataWriter() + serializer = SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()) + paths = tuple(f"images/{well}.tif" for well in ("A01", "A02", "A03", "A04")) + + def publish(item): + index, image_path = item + projection = calibrated_projection(f"A0{index + 1}", image_path, 0.5) + publish_projection_inventory( + writer, + path, + serializer.projection_fields(((projection, image_path),)), + (image_path,), + ) + + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(publish, enumerate(paths))) + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert set(subdir[FIELDS.WORKSPACE_MAPPING]) == set(paths) + assert len(subdir[FIELDS.IMAGE_FILES]) == 1 # Most recent partial step inventory. + assert subdir[FIELDS.PIXEL_SIZE] == 0.5 + publish_projection_inventory(writer, path, None, paths) + subdir = json.loads(path.read_text())[FIELDS.SUBDIRECTORIES]["images"] + assert subdir[FIELDS.IMAGE_FILES] == list(paths) + assert len(subdir[FIELDS.SOURCE_PROJECTION]) == len(paths) + assert subdir["wells"] == {f"A0{index + 1}": None for index in range(4)} + + def test_metadata_codec_rejects_unknown_declarations(): with pytest.raises(ValueError, match="undeclared"): ImagePayloadMetadata.from_mapping({"imagined_tile_axis": 9}) From aa07241fa1d3a6f3688494c89cd65de6c81e14cd Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 17:44:36 -0400 Subject: [PATCH 015/125] docs(perf): prioritize shared plumbing and record owner-level evidence (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 84 +++++++++++++++++-- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 9f09bf5cae..784ecf732e 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -6,6 +6,69 @@ rendering outside callable execution. This branch combines three production dependencies under their existing lifecycle owners. It does not yet establish an end-to-end speedup. +Primary target and abstraction admission +--------------------------------------- + +The primary performance goal is shared runtime work outside callable execution: +invocation and adapter preparation, stack loading, output registration, +finalization, saving, publication and provenance. Each existing abstraction must +own a distinct invariant, behavior or useful validated view. Objects that only +forward or reconstruct facts already owned elsewhere should be removed, with +consumers deriving directly from the determining owner. Shared algorithms belong +on existing nominal ancestors; a new class is not evidence of improved ownership. +Small measured redundancies are folded into the coherent migration even when +they cannot independently close the performance gap. + +The ordinary 3D probe at clean revision 5d2fd489946e6885f161a7379e4eec1d14e307fd +records 9.860550 seconds of public server execution. Its same-thread root takes +9.757393 seconds, including 4.097127 seconds at runtime callable boundaries and +5.118847 seconds elsewhere inside steps. Final plate publication adds 0.476819 +seconds. The callable boundary includes target resolution and signature filtering; +it is not an exact naked-kernel timer. Execution is inline for this one-worker +configuration; a configured fork policy does not imply a separate worker process. + +A deeper probe on the same source records 10.275668 seconds, including 0.023519 +seconds of explicitly isolated input capture. Its nonoverlapping coarse phases +locate 0.697612 seconds in output recording, 0.478745 in image-request preparation, +0.795071 in three atomic publication transactions and another 0.143612 in durable +target reconciliation. Remaining loading and output-registration work takes +0.739507 and 0.722780 seconds respectively. Stack metadata composition takes only +0.102824 seconds across all consumers. These diagnostic clocks locate work and +must not substitute for an ordinary uninstrumented paired comparison. + +Required output work remains included: this pipeline saves 120 TIFF planes and +six result CSVs. Some work outside the callable boundary is numerical table +assembly or actual saving, so the full noncallable duration is not asserted to be +removable indirection. Nine stack-composition calls belong to loading; nineteen +belong to unstacking. Aggregate composition counts do not establish cache misses. + +Saved real-payload replays confirm that duplicated input/name resolution and +rebuilding a compiled callable contract have small individual payoffs: about +0.112 milliseconds per input pair and 4.386 milliseconds per 211 resolutions. +These are included for ownership cleanup, without attributing the larger adapter +gap to them. The direct cache payload migration removes +``RuntimeImageStackCacheValue``, which owned only a forwarded ``stack`` field; +the cache retains path/memory identity, invalidation and execution lifetime. +All 102 related lifetime/source-projection controls pass for that migration. + +Complete saved atomic publication transactions retain actual parser, labels, +saved paths, pretransaction JSON, locking and filesystem writes. Deriving +geometry from the already decoded nominal projections reduces a 60-path +transaction from median 0.13556 to 0.07601 seconds and a 120-path transaction from +0.27224 to 0.15354 seconds, with byte-identical complete JSON. The production +implementation also reproduces both baseline byte hashes, and 57 metadata, +projection and image-output controls pass, including pruning, concurrent partial +inventory updates, fresh geometry and atomic rollback. This is stage replay +evidence, not an end-to-end speedup claim; integrated qualification remains in +progress. + +The existing CellProfiler executor now resolves the compiled raw callable once +at its validated invocation boundary and passes that target through the existing +processing and slice dispatch. The existing callable-view policy accepts that +already resolved target; no additional generic invocation type or parallel +contract/callable fields are needed. Image source-name projection consumes the +already resolved payload through the existing artifact strategy family. + The unmodified-main light trace located about 3.44 seconds in loading, unstacking, saving and finalization for the representative 3D pipeline. CP request and output context construction account for another approximately 1.216 seconds. These are @@ -138,15 +201,18 @@ source patch equivalence for squash/rebase history. No additional useful unpublished fix branch was identified. Rejected prototypes and obsolete planning branches are not nominated as unfinished production fixes. -The fresh open-issue audit verifies 7 of 29 have formal closing PR references; -22 still have no actual future fixing PR. Mere cross-references and diagnosis-only -drafts are not counted. Missing manual closing links were added for -138/207, 213/217, 320/217, 355/358 and 395/397. Existing 384/394 and 386/388 links -have closing cross-reference events. GitHub's official manual mutation is a no-op -for these already linked pairs; no exact ConnectedEvent is claimed for them. -Audit evidence: current-open-issues-connected-closing-audit-20261001.json/.md, -remaining-remote-branch-pr-audit-20261001.json and -local-worktree-branch-inventory-20261001.json in the external evidence directory. +The subsequent complete audit verifies eight of 26 open issues have formal +closing PR references and exact manual ConnectedEvent entries; 18 still lack an +actual complete fixing PR. Mere cross-references and diagnosis-only drafts are +not counted. The pairs are 138/207, 213/217, 320/217, 355/358, 384/394, 386/388, +395/397 and 398/399. For the three automatically detected links whose manual +mutation was initially a no-op, the exact original PR bodies were restored after +the manual links were created. Head, base and draft state were preserved. +Audit evidence: official-development-link-final-20261001.json/.md, +manual-development-relink-exact-events-v3-20261001.json, +remaining-remote-branch-pr-audit-20261001.json and local-worktree-branch-inventory- +20261001.json in the external evidence directory. Counts describe that pinned +audit rather than future repository state. The earlier runtime-plumbing prototype 2253ec63b is rejected. Its paired timing did not establish a gain and its original R0 gate failed. It is not part of this From 1b53d09f36e7bc38aa49109eecdbb6cb5418a0f1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 17:47:23 -0400 Subject: [PATCH 016/125] perf(runtime): reuse compiled callable and resolved image inputs (#384) --- .../cellprofiler/runtime/artifact_binding.py | 26 +++- .../runtime/function_contract_execution.py | 19 ++- .../cellprofiler/runtime/module_execution.py | 7 +- ...ellprofiler_function_contract_execution.py | 122 +++++++++++++++++- .../test_cellprofiler_module_execution.py | 32 ++++- 5 files changed, 192 insertions(+), 14 deletions(-) diff --git a/openhcs/interop/cellprofiler/runtime/artifact_binding.py b/openhcs/interop/cellprofiler/runtime/artifact_binding.py index 0fbfc14e41..34f1dc72b1 100644 --- a/openhcs/interop/cellprofiler/runtime/artifact_binding.py +++ b/openhcs/interop/cellprofiler/runtime/artifact_binding.py @@ -181,6 +181,14 @@ def source_image_name( del request return None + def source_image_name_from_value( + self, + value: RuntimeCallableArgument, + ) -> str | None: + """Project a source name from an already resolved artifact value.""" + del value + return None + def source_image_payload( self, request: RuntimeArtifactInputRequest, @@ -253,11 +261,15 @@ def runtime_input_value( def source_image_name( self, request: RuntimeArtifactInputRequest, + ) -> str | None: + return self.source_image_name_from_value(self.raw_runtime_input_value(request)) + + def source_image_name_from_value( + self, + value: RuntimeCallableArgument, ) -> str | None: return single_source_name( - image_payload_metadata( - self.raw_runtime_input_value(request) - ).source_provenance.represented_source_image_names + image_payload_metadata(value).source_provenance.represented_source_image_names ) def source_image_payload( @@ -331,7 +343,13 @@ def source_image_name( self, request: RuntimeArtifactInputRequest, ) -> str | None: - return self.object_labels(request).source_image_name + return self.source_image_name_from_value(self.object_labels(request)) + + def source_image_name_from_value( + self, + value: RuntimeCallableArgument, + ) -> str | None: + return cast(ObjectLabelSet, value).source_image_name def source_image_payload( self, diff --git a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py index c1279ed9cd..79e9a04279 100644 --- a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py +++ b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py @@ -62,7 +62,8 @@ ) _CELLPROFILER_RUNTIME_CALLABLE_POLICY = RuntimeCallablePolicy( - callable_view=RuntimeCallableView.RAW, + # execute() resolves the compiled raw target before processing dispatch. + callable_view=RuntimeCallableView.DECORATED, kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED, ) @@ -112,6 +113,7 @@ def execute( executor = type(self)(plane_projection=plane_projection) function_name = callable_contract.function_name processing_contract = callable_contract.require_processing_contract() + runtime_func = callable_contract.resolve_raw_runtime_callable() mode = execution_mode CellProfilerRuntimeProfileLogger.log_module_profile( "cp_executor_mode_resolution", @@ -131,12 +133,19 @@ def execute( | ProcessingContract.FLEXIBLE | ProcessingContract.VOLUMETRIC_TO_SLICE, ): + runtime_kwargs = dict(kwargs) + runtime_kwargs.update( + processing_contract.declaration.consume_semantic_controls( + runtime_kwargs, + func=func, + ) + ) result = processing_contract.execute( executor, - func, + runtime_func, image, callable_contract=callable_contract, - **dict(kwargs), + **runtime_kwargs, ) case ( ImagePayloadExecutionMode.FULL_STACK, @@ -146,7 +155,7 @@ def execute( | ProcessingContract.VOLUMETRIC_TO_SLICE, ): result = executor.execute_pure_3d( - func, + runtime_func, image, callable_contract=callable_contract, **dict(kwargs), @@ -160,7 +169,7 @@ def execute( ): result = executor._execute_aligned_multi_image_stack( callable_contract, - func, + runtime_func, image, **dict(kwargs), ) diff --git a/openhcs/interop/cellprofiler/runtime/module_execution.py b/openhcs/interop/cellprofiler/runtime/module_execution.py index 840624bfaf..54c6661583 100644 --- a/openhcs/interop/cellprofiler/runtime/module_execution.py +++ b/openhcs/interop/cellprofiler/runtime/module_execution.py @@ -1332,8 +1332,11 @@ def _image_request( input_binding = replace(input_binding, current_image=current_runtime_payload) for spec in image_inputs: request = input_binding.artifact_request_for_spec(spec) - payloads.append(image_strategy.runtime_input_value(request)) - source_names.append(image_strategy.source_image_name(request)) + payload = image_strategy.runtime_input_value(request) + payloads.append(payload) + source_names.append( + image_strategy.source_image_name_from_value(payload) + ) parameter_image_inputs = input_binding.image_inputs broadcast_sources = tuple( sources[0] diff --git a/tests/unit/test_cellprofiler_function_contract_execution.py b/tests/unit/test_cellprofiler_function_contract_execution.py index 2a61995cec..af8e2adff9 100644 --- a/tests/unit/test_cellprofiler_function_contract_execution.py +++ b/tests/unit/test_cellprofiler_function_contract_execution.py @@ -3,6 +3,9 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import dataclass, replace +from functools import wraps +import inspect import numpy as np import pytest @@ -18,7 +21,11 @@ ImageArtifactType, MeasurementsArtifactType, ) -from openhcs.core.callable_contract import CallableContract, CallableMetadata +from openhcs.core.callable_contract import ( + CallableContract, + CallableMetadata, + callable_request, +) from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, RuntimePlaneAxisValueProjection, @@ -76,6 +83,119 @@ def test_morphological_skeleton_executes_one_planar_runtime_image() -> None: assert image_payload_data(result).shape == image.shape +def test_compiled_slice_execution_resolves_raw_target_once_and_preserves_request_binding( + monkeypatch: pytest.MonkeyPatch, +) -> None: + @dataclass(frozen=True, slots=True) + class ScaleRequest: + image: object + scale: int + + seen: list[tuple[int, ...]] = [] + + @callable_request(ScaleRequest) + def request_bound(request: ScaleRequest) -> object: + data = image_payload_data(request.image) + seen.append(data.shape) + return data * request.scale + + @wraps(request_bound) + def decorated(*args: object, **kwargs: object) -> object: + raise AssertionError("CellProfiler invoked an outer runtime wrapper") + + contract = CallableContract.from_callable(decorated) + contract = replace( + contract, + metadata=replace( + contract.metadata, + processing_contract=ProcessingContract.PURE_2D, + ), + ) + raw_target = contract.resolve_canonical_raw_callable() + resolve_raw = CallableContract.resolve_raw_runtime_callable + resolutions: list[CallableContract] = [] + + def resolve_once(self: CallableContract) -> Callable[..., object]: + resolutions.append(self) + return resolve_raw(self) + + def reject_reconstruction(*args: object, **kwargs: object) -> CallableContract: + raise AssertionError("Runtime execution rebuilt its compiled contract") + + monkeypatch.setattr(CallableContract, "resolve_raw_runtime_callable", resolve_once) + monkeypatch.setattr(CallableContract, "from_callable", reject_reconstruction) + image = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE + ).payload_with(np.ones((3, 4, 5), dtype=np.float32), None) + + result = CellProfilerFunctionContractExecutor().execute( + contract, + raw_target, + image, + {"scale": 4, "adapter_control": 17}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=3, + ), + ) + + assert resolutions == [contract] + assert seen == [(4, 5)] * 3 + assert isinstance(result, RuntimeSliceAlignedValues) + np.testing.assert_array_equal( + np.stack(tuple(image_payload_data(value) for value in result.slices)), + np.full((3, 4, 5), 4.0), + ) + + +@pytest.mark.parametrize("canonical_default", (False, True)) +def test_flexible_dispatch_preserves_canonical_signature_control_defaults( + canonical_default: bool, +) -> None: + seen: list[tuple[tuple[int, ...], bool]] = [] + + def raw(image: np.ndarray, *, slice_by_slice: bool = False) -> np.ndarray: + seen.append((image.shape, slice_by_slice)) + return image + + @wraps(raw) + def decorated(*args: object, **kwargs: object) -> object: + raise AssertionError("CellProfiler invoked an outer runtime wrapper") + + signature = inspect.signature(raw) + decorated.__signature__ = signature.replace( + parameters=( + parameter.replace(default=canonical_default) + if parameter.name == "slice_by_slice" + else parameter + for parameter in signature.parameters.values() + ) + ) + contract = _compiled_contract(decorated, ProcessingContract.FLEXIBLE) + image = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE + ).payload_with(np.ones((3, 4, 5), dtype=np.float32), None) + + CellProfilerFunctionContractExecutor().execute( + contract, + decorated, + image, + {}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=3, + ), + ) + + assert seen == ( + [((4, 5), True)] * 3 + if canonical_default + else [((3, 4, 5), False)] + ) + + @pytest.mark.parametrize("processing_contract", tuple(ProcessingContract)) @pytest.mark.parametrize( ("image_mode", "expected_call_count"), diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index 9ba979906e..99a4b572ba 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -1368,7 +1368,9 @@ def test_runtime_binding_uses_full_compiled_special_input_contract() -> None: np.testing.assert_array_equal(image_payload_data(topology_inputs[1]), 1.0) -def test_image_artifact_resolution_uses_declared_artifact_alias() -> None: +def test_image_artifact_resolution_uses_declared_artifact_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: spec = ArtifactSpec.input("IllumStain1", ImageArtifactType) source = ImagePayloadMetadata( source_image_provenance_planes=SourceImageProvenancePlanes.from_components( @@ -1394,6 +1396,17 @@ def test_image_artifact_resolution_uses_declared_artifact_alias() -> None: ).source_image_name(RuntimeArtifactInputRequest(spec=spec, value=source)) is None ) + strategy = RuntimeArtifactTypeStrategy.for_artifact_type(ImageArtifactType) + + def reject_second_resolution(*args, **kwargs): + raise AssertionError("Source-name projection resolved the image again") + + monkeypatch.setattr( + type(strategy), "raw_runtime_input_value", reject_second_resolution + ) + # The original name remains a contributor; the alias alone is not a + # unique represented source name. + assert strategy.source_image_name_from_value(payload) is None def test_main_flow_image_artifact_selects_declared_alias_plane() -> None: @@ -1590,7 +1603,9 @@ def test_main_flow_image_artifact_preserves_declared_runtime_slice_stack() -> No assert image_payload_metadata(payload).source_image_names == ("Stain1",) -def test_main_flow_projection_binds_singleton_broadcast_artifact_as_2d() -> None: +def test_main_flow_projection_binds_singleton_broadcast_artifact_as_2d( + monkeypatch: pytest.MonkeyPatch, +) -> None: original_spec = ArtifactSpec.input("OrigStain1", ImageArtifactType) illumination_spec = ArtifactSpec.input( "IllumStain1", @@ -1643,12 +1658,25 @@ def test_main_flow_projection_binds_singleton_broadcast_artifact_as_2d() -> None executor = _module_executor(contract) _activate_runtime_contract(executor.callable_contract, runtime) + strategy_type = type( + RuntimeArtifactTypeStrategy.for_artifact_type(ImageArtifactType) + ) + resolve_image = strategy_type.raw_runtime_input_value + resolved_specs: list[ArtifactSpec] = [] + + def record_resolution(self, request: RuntimeArtifactInputRequest): + resolved_specs.append(request.spec) + return resolve_image(self, request) + + monkeypatch.setattr(strategy_type, "raw_runtime_input_value", record_resolution) + image_request = executor._image_request( current_image, runtime, module_type=CorrectIlluminationApplyModule, active_input_specs=contract.artifact_inputs.specs, ) + assert resolved_specs == [original_spec] runtime_kwargs = executor._runtime_input_kwargs( runtime, current_image, From bbc75a380d950f24b526bab8ca63220d7a4eb6ca Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 18:02:10 -0400 Subject: [PATCH 017/125] refactor: derive broadcast and output contexts from existing owners Move pure input-relation resolution to ArtifactSpecCollection and canonical output contextualization to RuntimeReturnedOutputMatcher. Delete duplicate CP executor helpers and retain composed measurement aliases from their declared input source. Refs #384. Validation: 454 focused artifact/output/CellProfiler tests pass; fresh import smoke and diff checks pass. --- openhcs/core/artifacts.py | 31 ++++++++ openhcs/core/runtime_output_matching.py | 76 ++++++++++++++++++ .../runtime/function_contract_execution.py | 77 +------------------ .../cellprofiler/runtime/module_execution.py | 43 ++--------- tests/unit/test_artifact_spec_collection.py | 42 ++++++++++ .../test_cellprofiler_module_execution.py | 50 ++++++++++++ tests/unit/test_runtime_output_matching.py | 58 ++++++++++++++ 7 files changed, 265 insertions(+), 112 deletions(-) diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index 04755abaa8..7078339c22 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -2110,6 +2110,37 @@ def __iter__(self): def __getitem__(self, index): return self.specs[index] + def stack_broadcast_source_indices(self) -> tuple[int | None, ...]: + """Resolve exact stack-broadcast owners from declared input relations.""" + + indices_by_ref: dict[ArtifactSpecRef, list[int]] = {} + for input_index, spec in enumerate(self.specs): + indices_by_ref.setdefault(spec.ref(), []).append(input_index) + + result: list[int | None] = [] + for input_index, spec in enumerate(self.specs): + sources = spec.stack_broadcast_sources() + if len(sources) > 1: + raise ValueError( + f"Input {spec.ref()!r} declares multiple stack-broadcast " + f"owners: {sources!r}." + ) + if not sources: + result.append(None) + continue + source_indices = tuple(indices_by_ref.get(sources[0], ())) + if len(source_indices) != 1: + raise ValueError( + f"Input {spec.ref()!r} requires exactly one active occurrence " + f"of stack-broadcast owner {sources[0]!r}, got " + f"{source_indices!r}." + ) + source_index = source_indices[0] + if source_index == input_index: + raise ValueError(f"Input {spec.ref()!r} cannot broadcast from itself.") + result.append(source_index) + return tuple(result) + def of_artifact_type( self, artifact_type: ArtifactTypeValue, diff --git a/openhcs/core/runtime_output_matching.py b/openhcs/core/runtime_output_matching.py index 44cc127d39..74a02ca0a7 100644 --- a/openhcs/core/runtime_output_matching.py +++ b/openhcs/core/runtime_output_matching.py @@ -9,6 +9,7 @@ from openhcs.core.aligned_image_payload import ( AlignedImageSliceContext, AlignedImageStack, + pack_aligned_image_outputs, ) from openhcs.core.artifacts import ( ArtifactOutputPlan, @@ -16,6 +17,12 @@ ArtifactSpecRef, ) from openhcs.core.callable_contract import CallableContract +from openhcs.core.runtime_image_values import image_payload_metadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxisValueProjection +from openhcs.core.runtime_slice_projection import ( + RuntimeSliceProjection, + RuntimeSliceProjectionDeclarationError, +) class RuntimeOutputBundle(ABC): @@ -61,6 +68,75 @@ def canonical_output(self) -> Any: return split_runtime_output(self.returned_output)[0] + def contextualize_canonical_output( + self, + *, + plane_projection: RuntimePlaneAxisValueProjection | None = None, + ) -> Any: + """Attach the compiled canonical ABI to one exact returned image axis.""" + + canonical_specs = self.callable_contract.canonical_return_output_specs.specs + if len(canonical_specs) <= 1: + return self.returned_output + canonical_output, trailing_outputs = split_runtime_output(self.returned_output) + if isinstance(canonical_output, AlignedImageStack): + if canonical_output.slice_contexts: + return self.returned_output + output_values = canonical_output.slices + else: + projection = plane_projection + function_name = self.callable_contract.function_name + if projection is None: + raise RuntimeSliceProjectionDeclarationError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs but returned a non-aligned payload without a " + "compiled plane projection." + ) + if projection.plane_index is not None: + raise RuntimeSliceProjectionDeclarationError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs after the compiled plane projection already selected " + f"plane {projection.plane_index}." + ) + if projection.axis_size != len(canonical_specs): + raise ValueError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs but its compiled plane projection declares " + f"{projection.axis_size} value(s)." + ) + output_axis = image_payload_metadata(canonical_output).plane_axis + if output_axis is not projection.axis: + raise RuntimeSliceProjectionDeclarationError( + f"{function_name} declares {len(canonical_specs)} canonical " + "outputs but its returned payload does not declare the " + f"compiled {projection.axis.value!r} plane axis; got " + f"{output_axis!r}." + ) + output_values = tuple( + RuntimeSliceProjection.value_for_slice( + canonical_output, + projection.selected_plane(output_index), + ) + for output_index in range(projection.axis_size) + ) + if len(output_values) != len(canonical_specs): + raise ValueError( + f"{self.callable_contract.function_name} returned {len(output_values)} " + "canonical output value(s) for " + f"{len(canonical_specs)} compiled output spec(s)." + ) + contextualized_output = pack_aligned_image_outputs( + output_values, + slice_contexts=AlignedImageSliceContext.main_flow_for_artifact_specs( + canonical_specs + ), + ) + return ( + (contextualized_output, *trailing_outputs) + if trailing_outputs + else contextualized_output + ) + def resolve(self) -> dict[ArtifactSpecRef, Any]: """Resolve every output in the callable ABI.""" diff --git a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py index 79e9a04279..7ce0d2275a 100644 --- a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py +++ b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py @@ -8,11 +8,9 @@ import numpy as np from openhcs.core.aligned_image_payload import ( - AlignedImageSliceContext, AlignedImageStack, ImagePayloadExecutionMode, aligned_image_stack_kwargs, - pack_aligned_image_outputs, ) from openhcs.core.callable_contract import CallableContract from openhcs.core.image_shapes import ArrayShape @@ -30,7 +28,7 @@ image_payload_mask, image_payload_metadata, ) -from openhcs.core.runtime_output_matching import split_runtime_output +from openhcs.core.runtime_output_matching import RuntimeReturnedOutputMatcher from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, RuntimePlaneAxisValueProjection, @@ -185,79 +183,10 @@ def execute( function=function_name, mode=mode.value, ) - return executor._contextualize_multi_canonical_output( + return RuntimeReturnedOutputMatcher( callable_contract, result, - ) - - def _contextualize_multi_canonical_output( - self, - callable_contract: CallableContract, - result: RuntimeCallableArgument, - ) -> RuntimeCallableArgument: - """Attach the compiled canonical ABI to one exact returned image axis.""" - - canonical_specs = callable_contract.canonical_return_output_specs.specs - if len(canonical_specs) <= 1: - return result - canonical_output, trailing_outputs = split_runtime_output(result) - if isinstance(canonical_output, AlignedImageStack): - if canonical_output.slice_contexts: - return result - output_values = canonical_output.slices - else: - projection = self.plane_projection - function_name = callable_contract.function_name - if projection is None: - raise RuntimeSliceProjectionDeclarationError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs but returned a non-aligned payload without a " - "compiled plane projection." - ) - if projection.plane_index is not None: - raise RuntimeSliceProjectionDeclarationError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs after the compiled plane projection already selected " - f"plane {projection.plane_index}." - ) - if projection.axis_size != len(canonical_specs): - raise ValueError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs but its compiled plane projection declares " - f"{projection.axis_size} value(s)." - ) - output_axis = image_payload_metadata(canonical_output).plane_axis - if output_axis is not projection.axis: - raise RuntimeSliceProjectionDeclarationError( - f"{function_name} declares {len(canonical_specs)} canonical " - "outputs but its returned payload does not declare the " - f"compiled {projection.axis.value!r} plane axis; got " - f"{output_axis!r}." - ) - output_values = tuple( - RuntimeSliceProjection.value_for_slice( - canonical_output, - projection.selected_plane(output_index), - ) - for output_index in range(projection.axis_size) - ) - if len(output_values) != len(canonical_specs): - raise ValueError( - f"{callable_contract.function_name} returned {len(output_values)} " - "canonical output value(s) for " - f"{len(canonical_specs)} compiled output spec(s)." - ) - contextualized_output = pack_aligned_image_outputs( - output_values, - slice_contexts=AlignedImageSliceContext.main_flow_for_artifact_specs( - canonical_specs - ), - ) - return ( - (contextualized_output, *trailing_outputs) - if trailing_outputs - else contextualized_output - ) + ).contextualize_canonical_output(plane_projection=executor.plane_projection) def execute_pure_2d_slice_batch( self, diff --git a/openhcs/interop/cellprofiler/runtime/module_execution.py b/openhcs/interop/cellprofiler/runtime/module_execution.py index 54c6661583..63c112c16b 100644 --- a/openhcs/interop/cellprofiler/runtime/module_execution.py +++ b/openhcs/interop/cellprofiler/runtime/module_execution.py @@ -1324,6 +1324,7 @@ def _image_request( adapter, ), ) + image_input_specs = ArtifactSpecCollection(image_inputs) payloads = [] source_names: list[str | None] = [] image_strategy = RuntimeArtifactTypeStrategy.for_artifact_type( @@ -1362,8 +1363,8 @@ def _image_request( if align_primary_images else () ), - stack_broadcast_source_indices=self._stack_broadcast_source_indices( - image_inputs + stack_broadcast_source_indices=( + image_input_specs.stack_broadcast_source_indices() ), ) return CellProfilerImageRequest( @@ -1371,12 +1372,12 @@ def _image_request( source_image_name=self._primary_image_source_name_from_sources( image_inputs, tuple(source_names) ), - source_aliases=ArtifactSpecCollection(image_inputs).names(), + source_aliases=image_input_specs.names(), image_count=len(payloads), execution_mode=composition.execution_mode, plane_projection=composition.preserved_plane_projection( adapter, - source_aliases=ArtifactSpecCollection(image_inputs).names(), + source_aliases=image_input_specs.names(), ), ) @@ -1413,40 +1414,6 @@ def _primary_image_source_name_from_sources( return None return source_names[0] - @staticmethod - def _stack_broadcast_source_indices( - input_specs: tuple[ArtifactSpec, ...], - ) -> tuple[int | None, ...]: - """Resolve exact stack-broadcast owners from declared input relations.""" - - indices_by_ref: dict[ArtifactSpecRef, list[int]] = {} - for input_index, spec in enumerate(input_specs): - indices_by_ref.setdefault(spec.ref(), []).append(input_index) - - result: list[int | None] = [] - for input_index, spec in enumerate(input_specs): - sources = spec.stack_broadcast_sources() - if len(sources) > 1: - raise ValueError( - f"Input {spec.ref()!r} declares multiple stack-broadcast " - f"owners: {sources!r}." - ) - if not sources: - result.append(None) - continue - source_indices = tuple(indices_by_ref.get(sources[0], ())) - if len(source_indices) != 1: - raise ValueError( - f"Input {spec.ref()!r} requires exactly one active occurrence " - f"of stack-broadcast owner {sources[0]!r}, got " - f"{source_indices!r}." - ) - source_index = source_indices[0] - if source_index == input_index: - raise ValueError(f"Input {spec.ref()!r} cannot broadcast from itself.") - result.append(source_index) - return tuple(result) - def _invocation_request( self, *, diff --git a/tests/unit/test_artifact_spec_collection.py b/tests/unit/test_artifact_spec_collection.py index db6e37dd1c..829bc2bcab 100644 --- a/tests/unit/test_artifact_spec_collection.py +++ b/tests/unit/test_artifact_spec_collection.py @@ -1,5 +1,7 @@ from __future__ import annotations +from dataclasses import replace + import pytest from openhcs.core.artifacts import ( @@ -9,6 +11,7 @@ ArtifactSpecCollection, GroupLineageSourceRelation, ImageArtifactType, + InputStackBroadcastSourceRelation, MainFlowPlaneProjectionOutputSpec, MeasurementsArtifactType, ObjectLabelsArtifactType, @@ -51,6 +54,45 @@ def test_artifact_spec_collection_queries_ordered_artifact_contracts() -> None: assert collection.by_name_and_artifact_type("Nuclei", ImageArtifactType) is None +def test_stack_broadcast_indices_use_exact_ordered_occurrences() -> None: + primary = ArtifactSpec.input("Primary", ImageArtifactType) + secondary = ArtifactSpec.input( + "Secondary", + ImageArtifactType, + relations=(InputStackBroadcastSourceRelation(source=primary.ref()),), + ) + unrelated = ArtifactSpec.input("Primary", ObjectLabelsArtifactType) + + assert ArtifactSpecCollection(()).stack_broadcast_source_indices() == () + assert ArtifactSpecCollection( + (secondary, unrelated, primary) + ).stack_broadcast_source_indices() == (2, None, None) + + for specs in ((secondary,), (primary, secondary, primary)): + with pytest.raises(ValueError, match="exactly one active occurrence"): + ArtifactSpecCollection(specs).stack_broadcast_source_indices() + + self_broadcast = replace( + primary, + relations=(InputStackBroadcastSourceRelation(source=primary.ref()),), + ) + with pytest.raises(ValueError, match="cannot broadcast from itself"): + ArtifactSpecCollection((self_broadcast,)).stack_broadcast_source_indices() + + other = ArtifactSpec.input("Other", ImageArtifactType) + multiple_owners = replace( + secondary, + relations=( + InputStackBroadcastSourceRelation(source=primary.ref()), + InputStackBroadcastSourceRelation(source=other.ref()), + ), + ) + with pytest.raises(ValueError, match="multiple stack-broadcast owners"): + ArtifactSpecCollection( + (primary, other, multiple_owners) + ).stack_broadcast_source_indices() + + def test_artifact_spec_collection_deduplicates_or_fails_loudly() -> None: image = ArtifactSpec.output("DNA", ImageArtifactType) diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index 99a4b572ba..90f612d3e4 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -50,6 +50,7 @@ from openhcs.core.callable_contract import ( CallableContract, CallableMetadata, + ImagePayloadConsumption, attach_callable_contract_metadata, runtime_image_execution_mode, ) @@ -1689,6 +1690,55 @@ def record_resolution(self, request: RuntimeArtifactInputRequest): assert image_payload_data(runtime_kwargs["illumination_function"]).shape == (4, 5) +def test_composed_measurement_images_keep_declared_aliases_and_resolved_payload() -> None: + image_spec = ArtifactSpec.input("DeclaredImage", ImageArtifactType) + contract = _compiled_callable_contract( + CellProfilerModule.require_module("MeasureObjectIntensity").require_callable(), + artifact_inputs=(image_spec,), + ) + contract = replace( + contract, + metadata=replace( + contract.metadata, + image_payload_consumption=ImagePayloadConsumption.COMPOSED, + ), + ) + image = np.ones((4, 5), dtype=np.float32) + runtime = _FakeCellProfilerRuntime( + {image_spec.name: image}, + artifact_input_edges=(_artifact_input_edge_for_test(image_spec),), + ) + executor = _module_executor(contract) + _activate_runtime_contract(contract, runtime) + image_request = CellProfilerImageRequest( + payload=image, + source_image_name="RuntimeAlias", + source_aliases=("RuntimeAlias",), + image_count=1, + execution_mode=ImagePayloadExecutionMode.NATURAL, + ) + + (measurement_image,) = executor._measurement_image_inputs( + runtime, + image, + image_request, + module_type=MeasureObjectIntensityModule, + ) + + assert measurement_image.source_aliases == (image_spec.name,) + assert measurement_image.payload is image_request.payload + assert measurement_image.execution_mode is image_request.execution_mode + assert measurement_image.plane_projection is image_request.plane_projection + assert not measurement_image.align_to_labels + with pytest.raises(ValueError, match="requires a composed measurement image request"): + executor._measurement_image_inputs( + runtime, + image, + None, + module_type=MeasureObjectIntensityModule, + ) + + def test_repeated_broadcast_inputs_consume_their_declared_source_group_axes() -> None: original_specs = tuple( ArtifactSpec.input(f"OrigStain{index}", ImageArtifactType) for index in (1, 2) diff --git a/tests/unit/test_runtime_output_matching.py b/tests/unit/test_runtime_output_matching.py index 74eb8a2e32..968ecb23ec 100644 --- a/tests/unit/test_runtime_output_matching.py +++ b/tests/unit/test_runtime_output_matching.py @@ -21,6 +21,12 @@ FunctionStepExecutionScope, ) from openhcs.core.runtime_output_matching import RuntimeReturnedOutputMatcher +from openhcs.core.runtime_image_values import ImagePayloadMetadata, image_payload_data +from openhcs.core.runtime_plane_projection import ( + RuntimePlaneAxis, + RuntimePlaneAxisValueProjection, +) +from openhcs.core.runtime_slice_projection import RuntimeSliceProjectionDeclarationError from openhcs.core.function_patterns import ( CompiledFunctionInvocation, FunctionInvocationKey, @@ -59,6 +65,58 @@ def test_runtime_output_matcher_maps_canonical_and_trailing_slots() -> None: } +def test_matcher_contextualizes_declared_axis_before_resolving_complete_abi() -> None: + first = ArtifactSpec.output("First", ImageArtifactType) + second = ArtifactSpec.output("Second", ImageArtifactType) + measurements = ArtifactSpec.output("Measurements", MeasurementsArtifactType) + data = np.arange(24, dtype=np.float32).reshape((2, 3, 4)) + payload = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE + ).payload_with(data, None) + trailing = object() + contract = _contract(first, second, measurements) + matcher = RuntimeReturnedOutputMatcher(contract, (payload, trailing)) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=2, + ) + + returned = matcher.contextualize_canonical_output(plane_projection=projection) + resolved = RuntimeReturnedOutputMatcher(contract, returned).resolve() + + assert returned[1] is trailing + assert resolved[measurements.ref()] is trailing + for index, spec in enumerate((first, second)): + selected = image_payload_data(resolved[spec.ref()]) + np.testing.assert_array_equal(selected, data[index]) + assert np.shares_memory(selected, data) + assert ( + RuntimeReturnedOutputMatcher(contract, returned).contextualize_canonical_output() + is returned + ) + + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="without a compiled"): + matcher.contextualize_canonical_output() + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="already selected"): + matcher.contextualize_canonical_output( + plane_projection=projection.selected_plane(0) + ) + with pytest.raises(ValueError, match="projection declares 3 value"): + matcher.contextualize_canonical_output( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, + axis_size=3, + ) + ) + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="plane axis"): + matcher.contextualize_canonical_output( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.SOURCE_BINDING, + axis_size=2, + ) + ) + + def test_runtime_output_matcher_uses_exact_multi_canonical_contexts() -> None: first = ArtifactSpec.output("First", ImageArtifactType) second = ArtifactSpec.output("Second", ImageArtifactType) From e82b2c072924d4c8d8161cab45ba0b0cc812ae31 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 18:07:39 -0400 Subject: [PATCH 018/125] docs: retain shared plumbing bounds and rejected cache assumptions Refs #384. Separate same-run diagnostic owner costs from ordinary paired timing and explain the existing metadata owner route. --- .../shared_runtime_plumbing_384_20261001.rst | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 784ecf732e..3e44bf056e 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -69,6 +69,34 @@ already resolved target; no additional generic invocation type or parallel contract/callable fields are needed. Image source-name projection consumes the already resolved payload through the existing artifact strategy family. +The executor does not own a second interpretation of input relations or returned +artifact contexts. ``ArtifactSpecCollection`` derives exact broadcast indices +from its ordered declarations; ``RuntimeReturnedOutputMatcher`` contextualizes +the compiled canonical return ABI. The old CP helpers are deleted. The existing +contract, collection and returned value determine these answers directly. + +The deeper clean-source probe at 5d1f1a781eda851b4783b68db342b76eeef9dea9 records +10.053237 seconds of public execution, including 4.465789 seconds at the callable +boundary and 4.939001 seconds elsewhere inside steps. Loading takes 0.699904 +exclusive seconds, image recording 0.519127, output identity 0.412492 and image +request preparation 0.388981. Explicit diagnostic capture costs 0.044853 seconds. +These partitions are diagnostic evidence, not accepted A/B measurements. + +All 23 produced-stack cache hits correspond to previously stored output paths. +The nine misses are original inputs; seven repeat earlier source reads and cost +0.5149 seconds inclusively. Those inputs are already in memory. Caching their +mutable arrays by path alone is rejected: storage has no mutation revision and +loaded source semantics depend on binding, aliases and plane contracts. + +The next shared route is metadata ownership, rather than a second pixel cache. +Changing spatial context, intensity or image names repeatedly constructs source +provenance and copies/fingerprints its component mapping. Source loading and CP +image recording consume this common path. The saved typed ImageMath fixture +retains actual nested mapping ownership for a cold-promotion and complete-workflow +replay. An isolated query-cache payoff does not admit this migration. CP-recorded +outputs already bypass generic postprocessing; removing that hypothetical second +contextualization would not affect the measured recorder. + The unmodified-main light trace located about 3.44 seconds in loading, unstacking, saving and finalization for the representative 3D pipeline. CP request and output context construction account for another approximately 1.216 seconds. These are From f3d798e39b27abac0a95ed8f329c246b21286df7 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 18:10:29 -0400 Subject: [PATCH 019/125] refactor: bind declared output contexts on aligned image stack AlignedImageStack owns complete exact-context matching; RuntimeReturnedOutputMatcher retains canonical/trailing ABI and projection handling. Delete the detached canonical-values helper and preserve all roster guards in one linear traversal. Refs #384. Validation: 475 focused controls pass; receiver-normalized algorithm AST retains original guards and messages. --- openhcs/core/aligned_image_payload.py | 52 ++++++++++++ openhcs/core/runtime_output_matching.py | 80 +++---------------- .../unit/test_aligned_main_flow_projection.py | 56 +++++++++++++ 3 files changed, 119 insertions(+), 69 deletions(-) diff --git a/openhcs/core/aligned_image_payload.py b/openhcs/core/aligned_image_payload.py index 23e33482c5..115ece9341 100644 --- a/openhcs/core/aligned_image_payload.py +++ b/openhcs/core/aligned_image_payload.py @@ -1194,6 +1194,58 @@ def projected_output_slices( for output_slice in payload_slices_for_alignment(payload): yield output_slice, context + def output_values_for_artifact_specs( + self, + canonical_specs: tuple[ArtifactSpec, ...], + ) -> dict[ArtifactSpecRef, Any]: + """Bind one complete declared main-flow roster to exact slice contexts.""" + + if not self.slice_contexts: + raise ValueError( + "Multiple canonical output specs require exact AlignedImageStack " + "slice contexts; positional slice order is not artifact identity." + ) + + specs_by_context = { + (spec.name, spec.artifact_type.value): spec for spec in canonical_specs + } + if len(specs_by_context) != len(canonical_specs): + raise ValueError("Canonical output ABI contains duplicate named contexts.") + resolved: dict[ArtifactSpecRef, Any] = {} + for payload, context in zip( + self.slices, + self.slice_contexts, + strict=True, + ): + if context.output_kind != AlignedImageSliceContext.MAIN_FLOW_OUTPUT_KIND: + raise ValueError( + "Canonical AlignedImageStack contains a non-main-flow slice " + f"context: {context!r}." + ) + context_key = (context.output_key, context.artifact_kind) + spec = specs_by_context.get(context_key) + if spec is None: + raise ValueError( + "Canonical AlignedImageStack context is not declared by the " + f"callable ABI: {context!r}." + ) + ref = spec.ref() + if ref in resolved: + raise ValueError( + "Canonical AlignedImageStack contains duplicate context for " + f"{ref!r}." + ) + resolved[ref] = payload + missing = tuple( + spec.ref() for spec in canonical_specs if spec.ref() not in resolved + ) + if missing: + raise ValueError( + "Canonical AlignedImageStack does not carry every declared output: " + f"{missing!r}." + ) + return resolved + def output_payload( self, artifact_ref: ArtifactSpecRef, diff --git a/openhcs/core/runtime_output_matching.py b/openhcs/core/runtime_output_matching.py index 74a02ca0a7..7e6dd98b49 100644 --- a/openhcs/core/runtime_output_matching.py +++ b/openhcs/core/runtime_output_matching.py @@ -161,12 +161,18 @@ def resolve(self) -> dict[ArtifactSpecRef, Any]: strict=True, ) } - resolved.update( - self._canonical_values( - self.callable_contract.canonical_return_output_specs.specs, - canonical_output, + canonical_specs = self.callable_contract.canonical_return_output_specs.specs + if len(canonical_specs) == 1: + resolved[canonical_specs[0].ref()] = canonical_output + elif canonical_specs: + if not isinstance(canonical_output, AlignedImageStack): + raise TypeError( + "Multiple canonical output specs require an AlignedImageStack with " + "one exact named slice context per output." + ) + resolved.update( + canonical_output.output_values_for_artifact_specs(canonical_specs) ) - ) return resolved def resolve_plan_values( @@ -205,70 +211,6 @@ def resolve_plan_values( matched_outputs.append((plan, spec, returned_values[ref])) return returned_values, tuple(matched_outputs) - @classmethod - def _canonical_values( - cls, - canonical_specs: tuple[ArtifactSpec, ...], - canonical_output: Any, - ) -> dict[ArtifactSpecRef, Any]: - """Resolve named outputs represented by the one canonical return slot.""" - - if not canonical_specs: - return {} - if len(canonical_specs) == 1: - spec = canonical_specs[0] - return {spec.ref(): canonical_output} - if not isinstance(canonical_output, AlignedImageStack): - raise TypeError( - "Multiple canonical output specs require an AlignedImageStack with " - "one exact named slice context per output." - ) - if not canonical_output.slice_contexts: - raise ValueError( - "Multiple canonical output specs require exact AlignedImageStack " - "slice contexts; positional slice order is not artifact identity." - ) - - specs_by_context = { - (spec.name, spec.artifact_type.value): spec for spec in canonical_specs - } - if len(specs_by_context) != len(canonical_specs): - raise ValueError("Canonical output ABI contains duplicate named contexts.") - resolved: dict[ArtifactSpecRef, Any] = {} - for payload, context in zip( - canonical_output.slices, - canonical_output.slice_contexts, - strict=True, - ): - if context.output_kind != AlignedImageSliceContext.MAIN_FLOW_OUTPUT_KIND: - raise ValueError( - "Canonical AlignedImageStack contains a non-main-flow slice " - f"context: {context!r}." - ) - context_key = (context.output_key, context.artifact_kind) - spec = specs_by_context.get(context_key) - if spec is None: - raise ValueError( - "Canonical AlignedImageStack context is not declared by the " - f"callable ABI: {context!r}." - ) - ref = spec.ref() - if ref in resolved: - raise ValueError( - "Canonical AlignedImageStack contains duplicate context for " - f"{ref!r}." - ) - resolved[ref] = payload - missing = tuple( - spec.ref() for spec in canonical_specs if spec.ref() not in resolved - ) - if missing: - raise ValueError( - "Canonical AlignedImageStack does not carry every declared output: " - f"{missing!r}." - ) - return resolved - @staticmethod def _specs_by_ref( specs: tuple[ArtifactSpec, ...], diff --git a/tests/unit/test_aligned_main_flow_projection.py b/tests/unit/test_aligned_main_flow_projection.py index b4e4e05b03..0bc40ba4da 100644 --- a/tests/unit/test_aligned_main_flow_projection.py +++ b/tests/unit/test_aligned_main_flow_projection.py @@ -52,6 +52,62 @@ def _input_ref( return ArtifactSpec.input(name, artifact_type).ref() +@pytest.mark.parametrize("carrier_type", (AlignedImageStack, ImageOutputBundle)) +def test_aligned_stack_binds_complete_declared_output_roster( + carrier_type: type[AlignedImageStack], +) -> None: + first = ArtifactSpec.output("First", ImageArtifactType) + second = ArtifactSpec.output("Second", ImageArtifactType) + first_context, second_context = AlignedImageSliceContext.main_flow_for_artifact_specs( + (first, second) + ) + first_value = object() + carrier = carrier_type( + (None, first_value), + (second_context, first_context), + ) + + assert carrier.output_values_for_artifact_specs((first, second)) == { + first.ref(): first_value, + second.ref(): None, + } + with pytest.raises(ValueError, match="duplicate named contexts"): + carrier.output_values_for_artifact_specs((first, first)) + + +@pytest.mark.parametrize( + ("contexts", "message"), + ( + ((), "require exact AlignedImageStack"), + (("first", "first"), "contains duplicate context"), + (("first",), "does not carry every declared output"), + (("first", "unknown"), "context is not declared"), + (("first", "non-main"), "contains a non-main-flow"), + ), +) +def test_aligned_stack_rejects_incomplete_or_conflicting_output_rosters( + contexts: tuple[str, ...], + message: str, +) -> None: + first = ArtifactSpec.output("First", ImageArtifactType) + second = ArtifactSpec.output("Second", ImageArtifactType) + first_context, second_context = AlignedImageSliceContext.main_flow_for_artifact_specs( + (first, second) + ) + selected = { + "first": first_context, + "unknown": replace(second_context, output_key="Unknown"), + "non-main": replace(second_context, output_kind="artifact"), + } + carrier = AlignedImageStack( + tuple(object() for _ in contexts) if contexts else (object(), object()), + tuple(selected[name] for name in contexts), + ) + + with pytest.raises(ValueError, match=message): + carrier.output_values_for_artifact_specs((first, second)) + + def _named_carrier() -> tuple[AlignedImageStack, np.ndarray, np.ndarray]: mcherry = np.full((3, 4), 3.0, dtype=np.float32) gfp = np.full((3, 4), 2.0, dtype=np.float32) From d9c9f8fe4b7e0361509bc975e0839c36477f04d1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 18:14:07 -0400 Subject: [PATCH 020/125] docs: record existing-owner gates against synchronized main Refs #384. Preserve failed guards and qualify the 955-control and original R0/R1 checks at their frozen source revision. --- .../shared_runtime_plumbing_384_20261001.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 3e44bf056e..67dbff5668 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -72,8 +72,10 @@ already resolved payload through the existing artifact strategy family. The executor does not own a second interpretation of input relations or returned artifact contexts. ``ArtifactSpecCollection`` derives exact broadcast indices from its ordered declarations; ``RuntimeReturnedOutputMatcher`` contextualizes -the compiled canonical return ABI. The old CP helpers are deleted. The existing -contract, collection and returned value determine these answers directly. +the compiled canonical return ABI. ``AlignedImageStack`` binds the complete +declared output roster to its exact slice contexts in one linear traversal. The +old CP and canonical-values helpers are deleted. The existing contract, +collection and returned value determine these answers directly. The deeper clean-source probe at 5d1f1a781eda851b4783b68db342b76eeef9dea9 records 10.053237 seconds of public execution, including 4.465789 seconds at the callable @@ -144,6 +146,18 @@ source checks. The main synchronization passed 85 help/output/source-resolution controls. These source ratchets are scoped guards, not a formal proof of every dynamic binding. +The subsequent ownership/publication follow-up normally merges main +76a2d392056f4fa6cc74bc122a552edc32f24239 at candidate +84057fbefd0ad5469c766051e19b34f42e851102, including the declaration-owned +classification and singleton image-selection repairs. All 955 controls across +26 changed-path and relevant main-owner test files pass, with two existing +watershed warnings. Original unmodified R0 and R1 pass against that main, with +no metric budgets or exclusions changed. The R1 scan uses the current NRA +checkout's virtual environment and its original 120-second deadline. Earlier +failed owner-growth/context-inspection guards and the wrong-interpreter import +failure are retained. Evidence: shared-plumbing-final-local-gates-20261001.json +and the original R0/R1 logs in the external evidence directory. + The first complete saved-native gate exposed two illumination-correction publication failures: actual numeric NPY saves admitted directories without any raster images into metadata publication. The inherited raster-inventory admission From 01af3e031af46399e11a1375094925bf4d69ac1d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 18:25:27 -0400 Subject: [PATCH 021/125] docs: retain mixed current-main plumbing benchmark outcomes Refs #384. Preserve all ordinary ABBA observations and the unresolved Imaging Flow Cytometry regression; no broad optimization or native-ratio claim. --- .../shared_runtime_plumbing_384_20261001.rst | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 67dbff5668..611ecffb5c 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -194,6 +194,36 @@ External evidence: shared-runtime-plumbing-repaired-qualified-abba-summary- 20261001.json and every corresponding source/input/observation receipt in the openhcs-benchmark-runs evidence directory. +Existing-owner follow-up ordinary ABBA +------------------------------------- + +The synchronized follow-up compares main 76a2d392 against candidate d9c9f8fe4. +The candidate's production source is the locally qualified 84057fb revision; +subsequent changes only retain the qualification receipt. Two observations per +side use the same ordinary public path and retain every sample. In seconds: + +========================= ======================= ======================= +Case Mean execution Mean pipeline total +========================= ======================= ======================= +3D monolayer 11.173033 -> 9.979570 13.166303 -> 11.915196 +Imaging Flow Cytometry 16.847054 -> 18.891492 19.058024 -> 21.417949 +Advanced segmentation 8.802229 -> 8.616838 11.271471 -> 11.113117 +========================= ======================= ======================= + +3D execution improves 10.68% and advanced execution 2.11%, while Imaging Flow +Cytometry regresses 12.14%. Its candidate observations are 17.371243 and 20.411741 +seconds, compared with 17.554674 and 16.139433 for main. Compilation is also +inflated in the slower candidate sweep. These samples establish mixed evidence, +not a statistical significance or causal attribution; PR394 remains draft. + +All 24 ordered Imaging Flow Cytometry step occurrences align. Most of the extra +time is inside steps, distributed across the slower candidate sweep. Between-step +gaps remain about 0.017-0.023 seconds; execution outside the complete step span +does not explain the regression. Existing output comparisons and shared metadata +ownership investigation continue before promotion. Evidence: +shared-plumbing-owner-followup-qualified-abba-20261001.json and its summary, +source receipts and all per-step/progress observations. + Fresh native qualification and next dominant route ------------------------------------------------ From 94c20da113968325cb7ff8ee8088d840d41620a3 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 18:41:07 -0400 Subject: [PATCH 022/125] docs: retain output equality and current startup integration gates Refs #384. Consume already merged issue355 repair with recorded ZMQ dependency; retain source-pinned performance and exact ordinary output checks. --- .../shared_runtime_plumbing_384_20261001.rst | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 611ecffb5c..94faedbd04 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -224,6 +224,26 @@ ownership investigation continue before promotion. Evidence: shared-plumbing-owner-followup-qualified-abba-20261001.json and its summary, source receipts and all per-step/progress observations. +Across all six run pairs, Imaging Flow Cytometry's two header rows and all +1,800-by-527 measurement cells match exactly. All six 3D CSV schemas, rows and +cells match, as do all 120 TIFF arrays, shapes and dtypes. Ordinary output +inventories match. Outcome-wire differences are fresh execution UUIDs and actual +run-root path prefixes; no scientific values are present in those outcome views. +Advanced's database/properties were not numerically compared, so absence of +ordinary CSV/TIFF output does not establish its parity. Evidence: +shared-plumbing-owner-followup-abba-step-output-audit-20261001.json. + +The next normal synchronization integrates main +0d07909b07870557a04f125452dc6944db77cb6c, including PR358's startup feedback and +worker-budget repair, at candidate fdf50241aa9adc6d96bc9c530dcc9acd19692b64. +The recorded ZMQRuntime dependency 668edafcb5ee530a163377c936431ae1e8334e85 is +checked out in both trees and consumed through the shared editable environment. +All 498 preparation/startup/CP integration controls pass, with six skips and two +existing warnings; original R0/R1 pass with unchanged budgets and exclusions. +Issue355 is closed by its existing merged PR. The ABBA remains pinned to its +measured sources; no timing result is projected through this synchronization. +Evidence: shared-plumbing-main355-local-gates-20261001.json. + Fresh native qualification and next dominant route ------------------------------------------------ From faf61ad87c550fa2d8ef44313344db74b378b857 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 18:57:37 -0400 Subject: [PATCH 023/125] docs: record current main sync and metadata equality review (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 94faedbd04..d1ca5d920c 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -244,6 +244,21 @@ Issue355 is closed by its existing merged PR. The ABBA remains pinned to its measured sources; no timing result is projected through this synchronization. Evidence: shared-plumbing-main355-local-gates-20261001.json. +Subsequent normal synchronization consumes documentation/knowledge changes in +main PR405 and PR406, through e66500e7ae3804abca1396681c50e1cef5e9a11e. These merges +do not change production source or dependencies. All 23 knowledge-transfer controls +pass on the final merged declarations, with two existing warnings. Benchmark and source-gate +observations remain pinned above. + +Independent review of the isolated metadata-owner migration found the important +distinction between ordered selection-record equality and public mapping-content +equality. It also checked duplicate stringified keys, normalization error order, +output-extension insertion order and fields-only transport. That migration remains +outside this PR until the affected consumers preserve their existing semantics +and the complete production-owner replay and source gates pass. Review evidence: +source-metadata-owner-independent-review-20261001.json. No stage replay is promoted +as an end-to-end gain. + Fresh native qualification and next dominant route ------------------------------------------------ From a698d275fb8657c08c1e7417f60570ade7f40870 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 19:21:42 -0400 Subject: [PATCH 024/125] perf: retain source metadata on its nominal lifetime owner (#384) --- openhcs/core/component_group_scope.py | 7 +- openhcs/core/runtime_image_values.py | 22 +- openhcs/core/runtime_source_binding_cache.py | 12 +- openhcs/core/source_binding_selection.py | 101 +--- openhcs/core/source_binding_workspace.py | 4 +- openhcs/core/source_bindings.py | 34 +- openhcs/core/source_image_provenance.py | 75 ++- openhcs/core/source_matching.py | 77 +-- openhcs/core/source_metadata.py | 552 +++++++++++++++--- openhcs/core/source_projection.py | 32 +- openhcs/core/source_workspace_projection.py | 15 +- .../function_artifact_materialization.py | 2 +- .../core/steps/function_output_identity.py | 46 +- openhcs/core/steps/function_runtime.py | 2 +- openhcs/core/virtual_workspace_metadata.py | 55 +- .../interop/cellprofiler/analyst_export.py | 4 +- .../cellprofiler/spreadsheet_export.py | 5 +- .../test_source_store_code_mode_zmq.py | 6 +- ...test_runtime_source_resolution_snapshot.py | 50 +- tests/unit/test_source_binding_workspace.py | 28 +- .../test_source_image_provenance_hotpath.py | 6 +- .../test_source_metadata_live_mutation.py | 4 +- tests/unit/test_source_metadata_owner.py | 490 ++++++++++++++++ tests/unit/test_source_projection.py | 4 +- 24 files changed, 1192 insertions(+), 441 deletions(-) create mode 100644 tests/unit/test_source_metadata_owner.py diff --git a/openhcs/core/component_group_scope.py b/openhcs/core/component_group_scope.py index 78b18d51af..9a5ca314f5 100644 --- a/openhcs/core/component_group_scope.py +++ b/openhcs/core/component_group_scope.py @@ -10,6 +10,7 @@ from openhcs.constants.constants import AllComponents from openhcs.core.component_set import ComponentSet from openhcs.core.source_metadata import ( + SourceMetadataFields, SourceMetadataMapping, SourceMetadataScalar, SourceMetadataValue, @@ -467,7 +468,7 @@ def for_group_coordinate( def fixed_component_metadata( self, metadata: SourceMetadataMapping | None = None, - ) -> dict[str, SourceMetadataValue]: + ) -> SourceMetadataMapping: """Merge fixed execution coordinates into source component metadata.""" from openhcs.constants.constants import get_multiprocessing_axis @@ -476,7 +477,9 @@ def fixed_component_metadata( with_source_component_metadata, ) - merged: dict[str, SourceMetadataValue] = dict(metadata or {}) + merged = SourceMetadataFields.composition_snapshot( + metadata if metadata is not None else {} + ) fixed_values = ( (get_multiprocessing_axis(), self.axis_id), *self.fixed_component_values, diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index 3a55746a9f..9e9f03e92e 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -47,6 +47,8 @@ ) from openhcs.core.source_metadata import ( SourceMetadataScalar, + OwnedSourceMetadataFields, + SourceMetadataFields, SourceMetadataValue, SourceVoxelSpacing, SourceVoxelSpacingFields, @@ -1884,7 +1886,9 @@ def common_source_component_metadata( """Return source metadata shared by the composed payload.""" metadata_values = tuple(values) metadata_by_plane = tuple( - dict(metadata.source_component_metadata) + SourceMetadataFields.composition_snapshot( + metadata.source_component_metadata + ) for metadata in metadata_values if metadata.source_component_metadata is not None ) @@ -1915,7 +1919,21 @@ def common_source_component_metadata( ) if not common_metadata: return None - return MappingProxyType(common_metadata) + owner = ( + metadata_by_plane[0] + if metadata_by_plane + and all( + isinstance(metadata, OwnedSourceMetadataFields) + for metadata in metadata_by_plane + ) + else {} + ) + derived = SourceMetadataFields.derived_mapping(owner, common_metadata) + return ( + derived + if isinstance(derived, OwnedSourceMetadataFields) + else MappingProxyType(derived) + ) @staticmethod def common_metadata_value( diff --git a/openhcs/core/runtime_source_binding_cache.py b/openhcs/core/runtime_source_binding_cache.py index 0231210c85..8b074a55c7 100644 --- a/openhcs/core/runtime_source_binding_cache.py +++ b/openhcs/core/runtime_source_binding_cache.py @@ -12,7 +12,10 @@ SourceBindingRuntimeMetadataNormalizer, MetadataExtractionRule, ) -from openhcs.core.source_metadata import SourceMetadataMapping +from openhcs.core.source_metadata import ( + ResolvedSourceMetadataRecord, + SourceMetadataMapping, +) if TYPE_CHECKING: from openhcs.core.source_binding_selection import ( @@ -42,8 +45,7 @@ def from_projection( ) -> "RuntimeSourceResolutionSnapshot": """Resolve declared positions into one independently owned runtime view.""" from openhcs.core.source_binding_selection import ( - ResolvedSourceMetadataRecord, - SourceMetadataRecord, + DeclaredSourceMetadataRecord, SourcePatternResolutionContext, ) @@ -59,7 +61,7 @@ def from_projection( context, source_metadata_by_path=MappingProxyType( { - path: SourceMetadataRecord.from_mapping(metadata) + path: DeclaredSourceMetadataRecord.from_mapping(metadata) for path, metadata in normalized_metadata.items() } ), @@ -77,7 +79,7 @@ def from_projection( records = {} for path in paths: metadata = context.metadata_for_path(path) - records[path] = ResolvedSourceMetadataRecord.from_resolved_mapping( + records[path] = ResolvedSourceMetadataRecord.from_mapping( {} if metadata is None else metadata ) context = replace(context, source_metadata_by_path=MappingProxyType(records)) diff --git a/openhcs/core/source_binding_selection.py b/openhcs/core/source_binding_selection.py index fde043f4dc..329813c31f 100644 --- a/openhcs/core/source_binding_selection.py +++ b/openhcs/core/source_binding_selection.py @@ -3,7 +3,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Iterator from dataclasses import dataclass, field, replace from enum import Enum from functools import lru_cache @@ -26,7 +25,6 @@ SourceBindingMatchMethod, SourceBindingMatchPlan, SourceBindingRuntimeContext, - SourceBindingRuntimeMetadataNormalizer, SourceSetRole, ) from openhcs.core.source_image_provenance import ( @@ -34,8 +32,9 @@ SourceImageProvenance, ) from openhcs.core.source_metadata import ( + SourceMetadataFields, SourceMetadataMapping, - SourceMetadataRoleView, + SourceMetadataRecord, SourceMetadataValue, ) from openhcs.core.source_matching import ( @@ -79,31 +78,20 @@ def _cached_source_candidate_pattern_keys(pattern_path: str) -> tuple[str, ...]: @dataclass(frozen=True, slots=True, eq=False) -class SourceMetadataRecord(Mapping[str, SourceMetadataValue]): - """Normalized source metadata carried across source-binding selection.""" - - fields: tuple[tuple[str, SourceMetadataValue], ...] +class DeclaredSourceMetadataRecord(SourceMetadataRecord): + """Live declared metadata that still requires path-specific fallbacks.""" @classmethod - def from_mapping(cls, metadata: SourceMetadataMapping) -> "SourceMetadataRecord": - return DeclaredSourceMetadataRecord( - tuple((str(key), value) for key, value in metadata.items()) - ) + def from_mapping( + cls, metadata: SourceMetadataMapping + ) -> "DeclaredSourceMetadataRecord": + return cls(tuple((str(key), value) for key, value in metadata.items())) - @abstractmethod def resolve( self, path: str, parser: "FilenameParser", metadata_rules: tuple[MetadataExtractionRule, ...], - ) -> "SourceMetadataRecord | None": - """Return this path's metadata through its declared resolution lifetime.""" - - def resolve_with_fallback( - self, - path: str, - parser: "FilenameParser", - metadata_rules: tuple[MetadataExtractionRule, ...], ) -> "SourceMetadataRecord | None": """Fill parser and extraction-rule fields absent from declarations.""" metadata: dict[str, SourceMetadataValue] = {} @@ -132,71 +120,6 @@ def resolve_with_fallback( ) return self.from_mapping(metadata) if metadata else None - def __getitem__(self, key: str) -> SourceMetadataValue: - for field_key, value in self.fields: - if field_key == key: - return value - raise KeyError(key) - - def __iter__(self) -> Iterator[str]: - return (key for key, _value in self.fields) - - def __len__(self) -> int: - return len(self.fields) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, SourceMetadataRecord): - return NotImplemented - return self.fields == other.fields - - def __hash__(self) -> int: - return hash((self.fields,)) - - -@dataclass(frozen=True, slots=True, eq=False) -class DeclaredSourceMetadataRecord(SourceMetadataRecord): - """Live declared metadata that still requires path-specific fallbacks.""" - - def resolve( - self, - path: str, - parser: "FilenameParser", - metadata_rules: tuple[MetadataExtractionRule, ...], - ) -> SourceMetadataRecord | None: - return self.resolve_with_fallback(path, parser, metadata_rules) - - -@dataclass(frozen=True, slots=True, eq=False) -class ResolvedSourceMetadataRecord(SourceMetadataRecord): - """Immutable path-specific metadata resolved at a runtime snapshot boundary.""" - - def __post_init__(self) -> None: - object.__setattr__( - self, - "fields", - tuple( - ( - str(key), - SourceBindingRuntimeMetadataNormalizer.normalized_value(value), - ) - for key, value in self.fields - ), - ) - - @classmethod - def from_resolved_mapping( - cls, metadata: SourceMetadataMapping - ) -> "ResolvedSourceMetadataRecord": - return cls(tuple(metadata.items())) - - def resolve( - self, - path: str, - parser: "FilenameParser", - metadata_rules: tuple[MetadataExtractionRule, ...], - ) -> SourceMetadataRecord | None: - return self if self.fields else None - @dataclass(frozen=True, slots=True) class SourceMetadataCandidates: @@ -274,7 +197,7 @@ def from_sources( str(path): ( metadata if isinstance(metadata, SourceMetadataRecord) - else SourceMetadataRecord.from_mapping(metadata) + else DeclaredSourceMetadataRecord.from_mapping(metadata) ) for path, metadata in source_metadata_by_path.items() } @@ -342,9 +265,9 @@ def candidate_filter_paths(self, pattern: SourceCandidatePath) -> tuple[str, ... dict.fromkeys( source_filter_path for metadata in self.metadata_for_paths(resolution.metadata_paths()) - for source_filter_path in SourceMetadataRoleView( + for source_filter_path in SourceMetadataFields.source_filter_paths( metadata - ).source_filter_paths() + ) ) ) if source_filter_paths: @@ -518,7 +441,7 @@ def merged_metadata_for_paths( if path_metadata is not None: merge_source_metadata(metadata, path_metadata, path=path) if metadata: - return SourceMetadataRecord.from_mapping(metadata) + return DeclaredSourceMetadataRecord.from_mapping(metadata) return None def source_metadata_by_paths( diff --git a/openhcs/core/source_binding_workspace.py b/openhcs/core/source_binding_workspace.py index 02230380d8..3567219876 100644 --- a/openhcs/core/source_binding_workspace.py +++ b/openhcs/core/source_binding_workspace.py @@ -45,7 +45,7 @@ OriginalSourceMetadata, SourceFilterPathMetadata, SourceMetadataMapping, - SourceMetadataRoleView, + SourceMetadataFields, SourceMetadataScalar, SourceVoxelSpacing, SourceComponentProjectionStrategy, @@ -1373,7 +1373,7 @@ def _shared_candidate_metadata( value_sets_by_key: dict[str, set[object]] = {} counts_by_key: dict[str, int] = {} for candidate in candidates: - for key, value in SourceMetadataRoleView(candidate.metadata).scalar_items(): + for key, value in SourceMetadataFields.scalar_items(candidate.metadata): value_sets_by_key.setdefault(key, set()).add(value) counts_by_key[key] = counts_by_key.get(key, 0) + 1 candidate_count = len(candidates) diff --git a/openhcs/core/source_bindings.py b/openhcs/core/source_bindings.py index 7f2bf54e30..9b00e410dd 100644 --- a/openhcs/core/source_bindings.py +++ b/openhcs/core/source_bindings.py @@ -38,9 +38,9 @@ from openhcs.core.runtime_tabular_values import FieldSpec from openhcs.core.source_metadata import ( SourceMetadataIdentityItems, - SourceMetadataIdentityProjection, + SourceMetadataFields, + ResolvedSourceMetadataRecord, SourceMetadataMapping, - SourceMetadataRoleView, SourceMetadataScalar, SourceMetadataValue, SourceVoxelSpacing, @@ -2000,7 +2000,7 @@ def metadata_fields_for_realized_source_metadata( declared_names = frozenset(field.name for field in declared_fields) values_by_name: dict[str, list[SourceMetadataScalar]] = {} for metadata in realized_source_metadata: - for field_name, value in SourceMetadataRoleView(metadata).original_items(): + for field_name, value in SourceMetadataFields.original_items(metadata): if ( field_name not in declared_names and field_name not in excluded_names @@ -2360,31 +2360,13 @@ class SourceBindingRuntimeMetadataNormalizer: def normalized(self) -> Mapping[str, SourceMetadataMapping]: return MappingProxyType( { - str(path): MappingProxyType( - { - str(key): self.normalized_value(value) - for key, value in source_metadata_dict(metadata).items() - } + str(path): ResolvedSourceMetadataRecord.normalized_mapping( + source_metadata_dict(metadata) ) for path, metadata in self.source_metadata_by_path.items() } ) - @classmethod - def normalized_value(cls, value: SourceMetadataValue) -> SourceMetadataValue: - if isinstance(value, Mapping): - return MappingProxyType( - { - str(key): cls.normalized_scalar(nested_value) - for key, nested_value in value.items() - } - ) - return cls.normalized_scalar(value) - - @staticmethod - def normalized_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: - return source_metadata_scalar(value) - @dataclass(frozen=True) class SourceBindingRuntimeContext: @@ -2538,7 +2520,7 @@ def source_metadata_identity( cached = self._source_metadata_identity if cached is None: cached = tuple( - (path, SourceMetadataIdentityProjection(metadata).items()) + (path, SourceMetadataFields.identity_items(metadata)) for path, metadata in sorted(self.source_metadata_by_path.items()) ) object.__setattr__(self, "_source_metadata_identity", cached) @@ -2673,9 +2655,9 @@ def metadata_identity_for_paths( identity: list[tuple[str, SourceMetadataIdentityItems]] = [] for path in paths: if path in self.source_metadata_by_path: - metadata = SourceMetadataIdentityProjection( + metadata = SourceMetadataFields.identity_items( self.source_metadata_by_path[path] - ).items() + ) else: metadata = () identity.append((path, metadata)) diff --git a/openhcs/core/source_image_provenance.py b/openhcs/core/source_image_provenance.py index 519003cbb0..025d77b87f 100644 --- a/openhcs/core/source_image_provenance.py +++ b/openhcs/core/source_image_provenance.py @@ -20,6 +20,8 @@ from openhcs.core.source_metadata import ( SOURCE_PLANE_COUNT_FIELD, SOURCE_PLANE_INDEX_FIELD, + OwnedSourceMetadataFields, + SourceMetadataFields, SourceMetadataMapping, SourceMetadataScalar, SourceMetadataValue, @@ -69,20 +71,12 @@ def normalize_source_path(source_path: str | None) -> str | None: return str(source_path) -def _normalize_component_metadata( - metadata: SourceComponentMetadata | None, -) -> SourceComponentMetadata | None: - if metadata is None: - return None - return MappingProxyType(dict(metadata)) - - def _component_metadata_identity( metadata: SourceComponentMetadata | None, ) -> tuple[tuple[str, str], ...] | None: if metadata is None: return None - return tuple(sorted((str(key), repr(value)) for key, value in metadata.items())) + return SourceMetadataFields.provenance_identity_items(metadata) def _tuple_value(values: tuple[Any, ...], index: int) -> Any | None: @@ -101,7 +95,10 @@ class SourceImageIdentity: def __post_init__(self) -> None: self.path = normalize_source_path(self.path) - self.component_metadata = _normalize_component_metadata(self.component_metadata) + if self.component_metadata is not None: + self.component_metadata = SourceMetadataFields.readonly_snapshot( + self.component_metadata + ) self._identity = ( self.path, _component_metadata_identity(self.component_metadata), @@ -147,21 +144,10 @@ def component_metadata_with_missing_from( if fallback.component_metadata is None: return self.component_metadata - merged = dict(self.component_metadata) - for component in AllComponents: - if source_component_metadata_value(merged, component) is not None: - continue - fallback_value = source_component_metadata_raw_value( - fallback.component_metadata, - component, - ) - if fallback_value is not None: - merged[component.value] = fallback_value - if self.filename_extension is None: - extension = fallback.filename_extension - if extension is not None: - merged["extension"] = extension - return MappingProxyType(merged) + merged = SourceMetadataFields.with_missing_from( + self.component_metadata, fallback.component_metadata + ) + return SourceMetadataFields.readonly_snapshot(merged) def with_parsed_path_components( self, @@ -1505,18 +1491,15 @@ def component_metadata_for_plane( self, plane_index: int, ) -> SourceComponentMetadata: - metadata = { - **dict(self.scalar_metadata), - SOURCE_PLANE_INDEX_FIELD: str(plane_index), - SOURCE_PLANE_COUNT_FIELD: str(self.source_plane_count), - } - return MappingProxyType( - with_source_component_metadata( - metadata, - AllComponents.Z_INDEX, - self.z_index_for_plane(plane_index), - ) + metadata = SourceMetadataFields.with_fields( + SourceMetadataFields.composition_snapshot(self.scalar_metadata), + { + SOURCE_PLANE_INDEX_FIELD: str(plane_index), + SOURCE_PLANE_COUNT_FIELD: str(self.source_plane_count), + }, + components=((AllComponents.Z_INDEX, self.z_index_for_plane(plane_index)),), ) + return SourceMetadataFields.readonly_snapshot(metadata) def z_index_for_plane(self, plane_index: int) -> int: scalar_z_index = source_component_metadata_value( @@ -1585,7 +1568,9 @@ def common_source_component_metadata( } if not common_metadata: return None - return MappingProxyType(common_metadata) + return SourceMetadataFields.readonly_snapshot( + SourceMetadataFields.derived_mapping(consensus, common_metadata) + ) def source_component_metadata_consensus( @@ -1594,7 +1579,9 @@ def source_component_metadata_consensus( """Return every metadata field, nulling values that differ between planes.""" metadata_values = tuple( - dict(metadata) for metadata in metadata_by_plane if metadata is not None + SourceMetadataFields.composition_snapshot(metadata) + for metadata in metadata_by_plane + if metadata is not None ) if len(metadata_values) != len(metadata_by_plane): return None @@ -1618,7 +1605,17 @@ def source_component_metadata_consensus( and all(value == present_values[0] for value in present_values) else None ) - return MappingProxyType(consensus) + owner = ( + metadata_values[0] + if all( + isinstance(metadata, OwnedSourceMetadataFields) + for metadata in metadata_values + ) + else {} + ) + return SourceMetadataFields.readonly_snapshot( + SourceMetadataFields.derived_mapping(owner, consensus) + ) def common_source_path(paths: Sequence[str | None]) -> str | None: diff --git a/openhcs/core/source_matching.py b/openhcs/core/source_matching.py index 3c9d1e0d86..66108ec9fe 100644 --- a/openhcs/core/source_matching.py +++ b/openhcs/core/source_matching.py @@ -30,12 +30,13 @@ OriginalSourceMetadata, SourceFilterPathMetadata, SourceMetadataMapping, - SourceMetadataRoleView, + SourceMetadataFields, SourceMetadataScalar, SourceMetadataValue, SourceComponentProjectionStrategy, path_metadata_values_equivalent, source_metadata_field_identity, + source_metadata_component, source_metadata_dict, source_metadata_scalar, ) @@ -709,7 +710,7 @@ def overlay_source_metadata( for component, value in projected_components.items(): overlaid = with_source_component_metadata(overlaid, component, value) - for field, value in SourceMetadataRoleView(additions).scalar_items(): + for field, value in SourceMetadataFields.scalar_items(additions): if value is None: continue component = source_metadata_component(field) @@ -770,7 +771,7 @@ def source_metadata_value( key: str, ) -> SourceMetadataScalar: """Return one source-literal metadata value by its exact declared key.""" - return _source_metadata_lookup_projection(metadata).value(key) + return SourceMetadataFields.literal_value(metadata, key) def source_component_metadata_value( @@ -819,7 +820,7 @@ def semantic_source_metadata_value( field_identity = source_metadata_field_identity(field_name) semantic_values = tuple( value - for field, value in SourceMetadataRoleView(metadata).scalar_items() + for field, value in SourceMetadataFields.scalar_items(metadata) if value is not None and source_metadata_field_identity(str(field)) == field_identity ) @@ -846,14 +847,14 @@ def source_component_metadata_values( component: AllComponents, ) -> tuple[str, ...]: """Return all metadata values that semantically describe a component.""" - return _source_metadata_lookup_projection(metadata).component_values(component) + return SourceMetadataFields.component_values(metadata, component) def with_source_component_metadata( metadata: SourceMetadataMapping, component: AllComponents, value: SourceMetadataScalar, -) -> dict[str, SourceMetadataValue]: +) -> SourceMetadataMapping: """Return metadata with one canonical component value. Source schemas can carry CellProfiler spellings such as ``Well`` and OpenHCS @@ -861,15 +862,7 @@ def with_source_component_metadata( every semantic spelling so later normalized lookups cannot observe both the old and new component values. """ - return { - **{ - key: field_value - for key, field_value in metadata.items() - if key == ORIGINAL_SOURCE_METADATA_FIELD - or source_metadata_component(str(key)) is not component - }, - component.value: str(value), - } + return SourceMetadataFields.with_component(metadata, component, value) def source_metadata_values_equal( @@ -969,62 +962,10 @@ def constraint_matches_metadata( return any( metadata_value is not None and source_metadata_values_equal(str(metadata_value), value) - for metadata_value in SourceMetadataRoleView(metadata).scalar_values() + for metadata_value in SourceMetadataFields.scalar_values(metadata) ) -@lru_cache(maxsize=256) -def source_metadata_component(field: str) -> AllComponents | None: - """Return the nominal component owner of a metadata field.""" - - return SourceComponentProjectionStrategy.component_for_metadata_field(field) - - -@dataclass(frozen=True, slots=True) -class SourceMetadataLookupProjection: - """Lookup projection over the current contents of one source metadata mapping.""" - - scalar_items: tuple[tuple[str, SourceMetadataScalar], ...] - original_items: tuple[tuple[str, SourceMetadataScalar], ...] - - def value(self, key: str) -> SourceMetadataScalar: - """Return a source-literal metadata value by exact key.""" - for candidate_key, value in self.original_items: - if str(candidate_key) == key and value is not None: - return value - for candidate_key, value in self.scalar_items: - if str(candidate_key) == key and value is not None: - return value - return None - - def component_values(self, component: AllComponents) -> tuple[str, ...]: - """Return all metadata values that semantically describe a component.""" - values: list[str] = [] - for field, field_value in self.scalar_items: - field_text = str(field) - if field_text == component.value and field_value is not None: - values.append(str(field_value)) - for field, field_value in self.scalar_items: - field_text = str(field) - if ( - field_text != component.value - and source_metadata_component(field_text) is component - ) and field_value is not None: - values.append(str(field_value)) - return tuple(dict.fromkeys(values)) - - -def _source_metadata_lookup_projection( - metadata: SourceMetadataMapping, -) -> SourceMetadataLookupProjection: - """Derive a lookup projection from the current nominal metadata role view.""" - role_view = SourceMetadataRoleView(metadata) - return SourceMetadataLookupProjection( - scalar_items=role_view.scalar_items(), - original_items=role_view.original_items(), - ) - - def _require_filter_value(clause: SourceFilterClause) -> str: if clause.value is None: raise ValueError( diff --git a/openhcs/core/source_metadata.py b/openhcs/core/source_metadata.py index 63c7e4e049..cdc7d47641 100644 --- a/openhcs/core/source_metadata.py +++ b/openhcs/core/source_metadata.py @@ -3,13 +3,14 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, field from enum import Enum from functools import lru_cache from math import isfinite from pathlib import Path -from typing import ClassVar, TypeAlias +from types import MappingProxyType +from typing import ClassVar, Self, TYPE_CHECKING, TypeAlias, TypeVar from metaclass_registry import AutoRegisterMeta from zmqruntime.viewer_protocol import ViewerWireField @@ -17,6 +18,11 @@ from openhcs.constants.constants import AllComponents from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +if TYPE_CHECKING: + from openhcs.core.source_bindings import MetadataExtractionRule + from openhcs.microscopes.microscope_interfaces import FilenameParser + + ORIGINAL_SOURCE_METADATA_FIELD = "OpenHCSOriginalSourceMetadata" SOURCE_FILTER_PATHS_METADATA_FIELD = "OpenHCSSourceFilterPaths" SOURCE_PLANE_INDEX_FIELD = "source_plane_index" @@ -38,6 +44,476 @@ ] +_MetadataViewT = TypeVar("_MetadataViewT") + + +@dataclass(frozen=True, slots=True, eq=False) +class SourceMetadataFields(Mapping[str, SourceMetadataValue]): + """Common field projection algorithms with nominal lifetime-selected views.""" + + fields: tuple[tuple[str, SourceMetadataValue], ...] + + @classmethod + def from_mapping(cls, metadata: SourceMetadataMapping) -> Self: + return cls(tuple(metadata.items())) + + @classmethod + def normalized_value(cls, value: SourceMetadataValue) -> SourceMetadataValue: + if isinstance(value, Mapping): + return MappingProxyType( + { + str(key): cls.normalized_scalar(nested_value) + for key, nested_value in value.items() + } + ) + return cls.normalized_scalar(value) + + @classmethod + def normalized_mapping( + cls, metadata: SourceMetadataMapping + ) -> SourceMetadataMapping: + return MappingProxyType( + {str(key): cls.normalized_value(value) for key, value in metadata.items()} + ) + + @staticmethod + def normalized_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: + return source_metadata_scalar(value) + + def __getitem__(self, key: str) -> SourceMetadataValue: + for field_key, value in self.fields: + if field_key == key: + return value + raise KeyError(key) + + def __iter__(self) -> Iterator[str]: + return (key for key, _value in self.fields) + + def __len__(self) -> int: + return len(self.fields) + + @abstractmethod + def __eq__(self, other: object) -> bool: + """Compare through the declared record or mapping namespace.""" + + def _derived_view( + self, key: str | AllComponents, derive: Callable[[], _MetadataViewT] + ) -> _MetadataViewT: + return derive() + + @staticmethod + def _view( + metadata: SourceMetadataMapping, + key: str | AllComponents, + derive: Callable[[], _MetadataViewT], + ) -> _MetadataViewT: + if isinstance(metadata, SourceMetadataFields): + return metadata._derived_view(key, derive) + return derive() + + @classmethod + def scalar_items( + cls, metadata: SourceMetadataMapping + ) -> tuple[tuple[str, SourceMetadataScalar], ...]: + """Read current raw fields or the immutable owner's scalar projection.""" + return cls._view( + metadata, + "scalar_items", + lambda: tuple( + (str(key), value) + for key, value in metadata.items() + if key != ORIGINAL_SOURCE_METADATA_FIELD + and not isinstance(value, Mapping) + ), + ) + + @classmethod + def scalar_values( + cls, metadata: SourceMetadataMapping + ) -> tuple[SourceMetadataScalar, ...]: + return tuple(value for _key, value in cls.scalar_items(metadata)) + + @classmethod + def original_items( + cls, metadata: SourceMetadataMapping + ) -> tuple[tuple[str, SourceMetadataScalar], ...]: + def project() -> tuple[tuple[str, SourceMetadataScalar], ...]: + original_metadata = metadata.get(ORIGINAL_SOURCE_METADATA_FIELD) + if original_metadata is None: + return () + return OriginalSourceMetadata.from_reserved_value( + original_metadata, path=ORIGINAL_SOURCE_METADATA_FIELD + ).fields + + return cls._view(metadata, "original_items", project) + + @classmethod + def source_filter_paths(cls, metadata: SourceMetadataMapping) -> tuple[str, ...]: + def project() -> tuple[str, ...]: + paths = metadata.get(SOURCE_FILTER_PATHS_METADATA_FIELD) + if paths is None: + return () + return SourceFilterPathMetadata.from_reserved_value( + paths, path=SOURCE_FILTER_PATHS_METADATA_FIELD + ).paths + + return cls._view(metadata, "source_filter_paths", project) + + @classmethod + def identity_items( + cls, metadata: SourceMetadataMapping + ) -> SourceMetadataIdentityItems: + def project() -> SourceMetadataIdentityItems: + return tuple( + sorted( + ( + str(key), + ( + tuple(sorted((str(k), v) for k, v in value.items())) + if isinstance(value, Mapping) + else value + ), + ) + for key, value in metadata.items() + ) + ) + + return cls._view(metadata, "identity_items", project) + + @classmethod + def provenance_identity_items( + cls, metadata: SourceMetadataMapping + ) -> tuple[tuple[str, str], ...]: + """Preserve the exact stored repr, including nested mapping insertion order.""" + return cls._view( + metadata, + "provenance_identity_items", + lambda: tuple(sorted((str(k), repr(v)) for k, v in metadata.items())), + ) + + @classmethod + def literal_value( + cls, metadata: SourceMetadataMapping, key: str + ) -> SourceMetadataScalar: + # Preserve the existing lookup's role-validation order even when the + # requested literal could be read without the original metadata role. + scalars = cls.scalar_items(metadata) + original = cls.original_items(metadata) + for fields in (original, scalars): + for candidate_key, value in fields: + if str(candidate_key) == key and value is not None: + return value + return None + + @classmethod + def component_value( + cls, metadata: SourceMetadataMapping, component: AllComponents + ) -> str | None: + return cls._view( + metadata, + component, + lambda: SourceComponentProjectionStrategy.for_enum_member( + component + ).metadata_value(metadata), + ) + + @classmethod + def component_values( + cls, metadata: SourceMetadataMapping, component: AllComponents + ) -> tuple[str, ...]: + scalars = cls.scalar_items(metadata) + cls.original_items(metadata) + values = [ + str(value) + for field, value in scalars + if str(field) == component.value and value is not None + ] + values.extend( + str(value) + for field, value in scalars + if str(field) != component.value + and source_metadata_component(str(field)) is component + and value is not None + ) + return tuple(dict.fromkeys(values)) + + @staticmethod + def readonly_snapshot(metadata: SourceMetadataMapping) -> SourceMetadataMapping: + """Preserve the old mapping snapshot while retaining an owned record's lifetime.""" + if isinstance(metadata, SourceMetadataFields): + return metadata._readonly_snapshot() + return MappingProxyType(dict(metadata)) + + def _readonly_snapshot(self) -> SourceMetadataMapping: + return MappingProxyType(dict(self)) + + @staticmethod + def composition_snapshot(metadata: SourceMetadataMapping) -> SourceMetadataMapping: + """Capture one composition's raw fields; owned records already hold that snapshot.""" + if isinstance(metadata, SourceMetadataFields): + return metadata._composition_snapshot() + return dict(metadata) + + def _composition_snapshot(self) -> SourceMetadataMapping: + return dict(self) + + @staticmethod + def derived_mapping( + metadata: SourceMetadataMapping, + fields: dict[str, SourceMetadataValue] | None, + ) -> SourceMetadataMapping: + """Keep owned lifetime on derived fields; raw callers receive a fresh dict.""" + if isinstance(metadata, SourceMetadataFields): + return metadata._derive_fields(fields) + return dict(metadata) if fields is None else fields + + def _derive_fields( + self, fields: dict[str, SourceMetadataValue] | None + ) -> SourceMetadataMapping: + return dict(self) if fields is None else fields + + @classmethod + def with_fields( + cls, + metadata: SourceMetadataMapping, + updates: SourceMetadataMapping, + *, + without: Iterable[str] = (), + components: Iterable[tuple[AllComponents, SourceMetadataScalar]] = (), + after_components: SourceMetadataMapping | None = None, + ) -> SourceMetadataMapping: + excluded = frozenset(without) + fields = {key: value for key, value in metadata.items() if key not in excluded} + fields.update(updates) + for component, value in components: + fields = { + key: field_value + for key, field_value in fields.items() + if key == ORIGINAL_SOURCE_METADATA_FIELD + or source_metadata_component(str(key)) is not component + } + fields[component.value] = str(value) + if after_components is not None: + fields.update(after_components) + return cls.derived_mapping(metadata, fields) + + @classmethod + def with_component( + cls, + metadata: SourceMetadataMapping, + component: AllComponents, + value: SourceMetadataScalar, + ) -> SourceMetadataMapping: + return cls.with_fields(metadata, {}, components=((component, value),)) + + @classmethod + def with_missing_from( + cls, metadata: SourceMetadataMapping, fallback: SourceMetadataMapping + ) -> SourceMetadataMapping: + merged: dict[str, SourceMetadataValue] | None = None + current = metadata + for component in AllComponents: + if cls.component_value(current, component) is not None: + continue + value = cls.component_value(fallback, component) + if value is not None: + if merged is None: + merged = dict(metadata) + merged[component.value] = value + current = merged + if cls.literal_value(metadata, "extension") is None: + extension = cls.literal_value(fallback, "extension") + if extension is not None: + if merged is None: + merged = dict(metadata) + merged["extension"] = str(extension) + if merged is None: + return cls.derived_mapping(metadata, None) + return cls.derived_mapping(metadata, merged) + + +@dataclass(frozen=True, slots=True, eq=False) +class SourceMetadataRecord(SourceMetadataFields): + """Ordered source metadata carried through declared selector resolution.""" + + @abstractmethod + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple["MetadataExtractionRule", ...], + ) -> "SourceMetadataRecord | None": + """Resolve fields through their declared metadata lifetime.""" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SourceMetadataRecord): + return NotImplemented + return self.fields == other.fields + + def __hash__(self) -> int: + return hash((self.fields,)) + + +@dataclass(frozen=True, slots=True, eq=False) +class OwnedSourceMetadataFields(SourceMetadataFields): + """Deep field ownership, indexed lookup, bounded views and local transport lifetime.""" + + _field_index: Mapping[str, SourceMetadataValue] = field( + init=False, repr=False, compare=False + ) + _views: dict[str | AllComponents, object] = field( + default_factory=dict, init=False, repr=False, compare=False + ) + _cacheable: bool = field(init=False, repr=False, compare=False) + + def __post_init__(self) -> None: + fields = tuple( + (str(key), self.normalized_value(value)) for key, value in self.fields + ) + object.__setattr__(self, "fields", self._normalized_fields(fields)) + self._initialize_derived_views() + + @staticmethod + def _normalized_fields( + fields: tuple[tuple[str, SourceMetadataValue], ...], + ) -> tuple[tuple[str, SourceMetadataValue], ...]: + return fields + + def _initialize_derived_views(self) -> None: + object.__setattr__(self, "_views", {}) + index: dict[str, SourceMetadataValue] = {} + for key, value in self.fields: + index.setdefault(key, value) + object.__setattr__(self, "_field_index", MappingProxyType(index)) + object.__setattr__( + self, + "_cacheable", + all( + ( + all( + type(v) in (str, int, float, bool, type(None)) + for v in value.values() + ) + if isinstance(value, Mapping) + else type(value) in (str, int, float, bool, type(None)) + ) + for _key, value in self.fields + ), + ) + + def __reduce__(self): + fields = tuple( + (key, dict(value) if isinstance(value, Mapping) else value) + for key, value in self.fields + ) + return (type(self)._from_serialized_fields, (fields,)) + + @classmethod + def _from_serialized_fields( + cls, fields: tuple[tuple[str, SourceMetadataValue], ...] + ) -> Self: + """Restore stored spelling and ownership without serializing derived views.""" + record = cls.__new__(cls) + object.__setattr__( + record, + "fields", + tuple( + ( + key, + ( + MappingProxyType(dict(value)) + if isinstance(value, Mapping) + else value + ), + ) + for key, value in fields + ), + ) + record._initialize_derived_views() + return record + + def __getitem__(self, key: str) -> SourceMetadataValue: + return self._field_index[key] + + @property + def metadata_contents(self) -> SourceMetadataMapping: + """Read-only mapping contents, separate from ordered record equality.""" + return self._field_index + + def _derived_view( + self, key: str | AllComponents, derive: Callable[[], _MetadataViewT] + ) -> _MetadataViewT: + if not self._cacheable: + return derive() + if key not in self._views: + self._views[key] = derive() + return self._views[key] + + def _readonly_snapshot(self) -> SourceMetadataMapping: + return self + + def _composition_snapshot(self) -> SourceMetadataMapping: + return self + + def _derive_fields( + self, fields: dict[str, SourceMetadataValue] | None + ) -> SourceMetadataMapping: + return self if fields is None else DurableSourceMetadata.from_mapping(fields) + + +@dataclass(frozen=True, slots=True, eq=False) +class ResolvedSourceMetadataRecord(OwnedSourceMetadataFields, SourceMetadataRecord): + """Owned runtime fields with the existing ordered selector record contract.""" + + def _readonly_snapshot(self) -> SourceMetadataMapping: + return self._derived_view( + "readonly_snapshot", + lambda: DurableSourceMetadata.from_mapping(self._field_index), + ) + + def resolve( + self, + path: str, + parser: "FilenameParser", + metadata_rules: tuple["MetadataExtractionRule", ...], + ) -> SourceMetadataRecord | None: + return self if self.fields else None + + +@dataclass(frozen=True, slots=True, eq=False) +class DurableSourceMetadata(OwnedSourceMetadataFields): + """Owned literal metadata from durable storage and image field derivation.""" + + @staticmethod + def _normalized_fields( + fields: tuple[tuple[str, SourceMetadataValue], ...], + ) -> tuple[tuple[str, SourceMetadataValue], ...]: + return tuple(dict(fields).items()) + + def __eq__(self, other: object) -> bool: + if isinstance(other, SourceMetadataRecord): + return NotImplemented + return Mapping.__eq__(self, other) + + __hash__ = None + + @staticmethod + def normalized_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Mapping) or ( + isinstance(value, Sequence) and not isinstance(value, str) + ): + raise RuntimeError( + "virtual_workspace source metadata supports scalar values and " + "one-level scalar mappings only." + ) + raise RuntimeError( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + + def source_metadata_dict( metadata: SourceMetadataMapping, ) -> dict[str, SourceMetadataValue]: @@ -489,68 +965,6 @@ def normalize_source_voxel_spacing_fields(self) -> None: ) -@dataclass(frozen=True, slots=True) -class SourceMetadataRoleView: - """Role-aware metadata view for literal selectors versus component axes.""" - - metadata: SourceMetadataMapping - - def scalar_items(self) -> tuple[tuple[str, SourceMetadataScalar], ...]: - items: list[tuple[str, SourceMetadataScalar]] = [] - for key, value in self.metadata.items(): - if key == ORIGINAL_SOURCE_METADATA_FIELD or isinstance(value, Mapping): - continue - items.append((str(key), value)) - return tuple(items) - - def scalar_values(self) -> tuple[SourceMetadataScalar, ...]: - return tuple(value for _key, value in self.scalar_items()) - - def original_items(self) -> tuple[tuple[str, SourceMetadataScalar], ...]: - original_metadata = self.metadata.get(ORIGINAL_SOURCE_METADATA_FIELD) - if original_metadata is None: - return () - return OriginalSourceMetadata.from_reserved_value( - original_metadata, - path=ORIGINAL_SOURCE_METADATA_FIELD, - ).fields - - def source_filter_paths(self) -> tuple[str, ...]: - source_filter_paths = self.metadata.get(SOURCE_FILTER_PATHS_METADATA_FIELD) - if source_filter_paths is None: - return () - return SourceFilterPathMetadata.from_reserved_value( - source_filter_paths, - path=SOURCE_FILTER_PATHS_METADATA_FIELD, - ).paths - - -@dataclass(frozen=True, slots=True) -class SourceMetadataIdentityProjection: - """Stable hash projection for source metadata records.""" - - metadata: SourceMetadataMapping - - def items(self) -> SourceMetadataIdentityItems: - projected: list[tuple[str, SourceMetadataIdentityValue]] = [] - for key, value in self.metadata.items(): - if isinstance(value, Mapping): - projected.append( - ( - str(key), - tuple( - sorted( - (str(nested_key), nested_value) - for nested_key, nested_value in value.items() - ) - ), - ) - ) - continue - projected.append((str(key), value)) - return tuple(sorted(projected)) - - @lru_cache(maxsize=4096) def source_metadata_field_identity(field: str) -> str: """Return the canonical semantic identity of one source metadata field.""" @@ -565,6 +979,12 @@ def source_metadata_field_identity(field: str) -> str: ) +@lru_cache(maxsize=256) +def source_metadata_component(field: str) -> AllComponents | None: + """Return the nominal component owner of a metadata field.""" + return SourceComponentProjectionStrategy.component_for_metadata_field(field) + + class SourceComponentProjectionStrategy( EnumKeyedStrategyMixin[AllComponents], ABC, @@ -591,7 +1011,7 @@ def metadata_component( component: AllComponents, metadata: SourceMetadataMapping, ) -> str | None: - return cls.for_enum_member(component).metadata_value(metadata) + return SourceMetadataFields.component_value(metadata, component) @classmethod def component_for_metadata_field( @@ -625,7 +1045,7 @@ def _metadata_group_value( metadata: SourceMetadataMapping, group: tuple[str, ...], ) -> str | None: - scalar_items = SourceMetadataRoleView(metadata).scalar_items() + scalar_items = SourceMetadataFields.scalar_items(metadata) for alias in group: alias_identity = source_metadata_field_identity(alias) for field_name, value in scalar_items: diff --git a/openhcs/core/source_projection.py b/openhcs/core/source_projection.py index 1ef3bb0834..40c1321c60 100644 --- a/openhcs/core/source_projection.py +++ b/openhcs/core/source_projection.py @@ -34,9 +34,9 @@ ) from openhcs.core.source_metadata import ( SourceComponentProjectionStrategy, - SourceMetadataIdentityProjection, + SourceMetadataFields, + ResolvedSourceMetadataRecord, SourceMetadataMapping, - SourceMetadataRoleView, SourceMetadataValue, source_metadata_dict, source_metadata_scalar, @@ -448,7 +448,7 @@ def identity_key(self) -> tuple[object, ...]: self.declared_address, self.dataset_identity, self.store_identity, - SourceMetadataIdentityProjection(self.metadata).items(), + SourceMetadataFields.identity_items(self.metadata), ) @@ -1219,7 +1219,7 @@ def _source_metadata( metadata = source_metadata_dict(projection.source_metadata) source_component_fields = { field: value - for field, value in SourceMetadataRoleView(metadata).scalar_items() + for field, value in SourceMetadataFields.scalar_items(metadata) if ( (component := source_metadata_component(field)) is not None and field != component.value @@ -1231,7 +1231,7 @@ def _source_metadata( source_component_fields, path=projection.ref.backend_address, ) - original_metadata = dict(SourceMetadataRoleView(metadata).original_items()) + original_metadata = dict(SourceMetadataFields.original_items(metadata)) for component, value in projection.source_component_values(): canonical_value = metadata.get(component.value) conflicts_with_address = ( @@ -1303,21 +1303,6 @@ def _padded(value: str, width: int) -> str: return f"{int(value):0{width}d}" if value.isdecimal() else value -def _normalized_source_metadata_value( - value: SourceMetadataValue, -) -> SourceMetadataValue: - """Freeze one source-metadata value without erasing its nominal shape.""" - - if isinstance(value, Mapping): - return MappingProxyType( - { - str(key): source_metadata_scalar(nested_value) - for key, nested_value in value.items() - } - ) - return source_metadata_scalar(value) - - def _normalize_projection(projection: SourceProjection) -> None: """Normalize fields shared by every nominal source projection.""" @@ -1338,12 +1323,7 @@ def _normalize_projection(projection: SourceProjection) -> None: object.__setattr__( projection, "source_metadata", - MappingProxyType( - { - str(key): _normalized_source_metadata_value(value) - for key, value in projection.source_metadata.items() - } - ), + ResolvedSourceMetadataRecord.normalized_mapping(projection.source_metadata), ) object.__setattr__( projection, diff --git a/openhcs/core/source_workspace_projection.py b/openhcs/core/source_workspace_projection.py index 84878b8851..8caef9e15f 100644 --- a/openhcs/core/source_workspace_projection.py +++ b/openhcs/core/source_workspace_projection.py @@ -23,7 +23,10 @@ SOURCE_BINDING_ALIAS_METADATA_FIELD, SourceProjectionRole, ) -from openhcs.core.source_metadata import SourceMetadataMapping +from openhcs.core.source_metadata import ( + SourceMetadataFields, + SourceMetadataMapping, +) from openhcs.core.source_matching import ( source_component_metadata_values, source_metadata_value, @@ -531,12 +534,8 @@ def apply(self, payload: RuntimeArrayData) -> RuntimeArrayData: """Apply declared component metadata and source aliases to one payload.""" source_metadata = self.source_metadata if source_metadata is not None: - source_metadata = MappingProxyType( - { - field: value - for field, value in source_metadata.items() - if field != SOURCE_BINDING_ALIAS_METADATA_FIELD - } + source_metadata = SourceMetadataFields.with_fields( + source_metadata, {}, without=(SOURCE_BINDING_ALIAS_METADATA_FIELD,) ) current_metadata = image_payload_metadata(payload) metadata = self.metadata(current_metadata) @@ -817,7 +816,7 @@ def record_source_metadata( virtual_path: str, metadata_fields: SourceMetadataMapping, ) -> None: - normalized_metadata = MappingProxyType(dict(metadata_fields)) + normalized_metadata = SourceMetadataFields.readonly_snapshot(metadata_fields) self.source_metadata_by_path[virtual_path] = normalized_metadata self.source_metadata_by_path[str(self.plate_path / virtual_path)] = ( normalized_metadata diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index 7510f0f61d..35effbb957 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -758,7 +758,7 @@ def record_metadata_with_runtime_scope( ) elif scope.component is not None: component_metadata = with_source_component_metadata( - dict(metadata.source_component_metadata or {}), + metadata.source_component_metadata or {}, scope.component, scope.require_value_text(), ) diff --git a/openhcs/core/steps/function_output_identity.py b/openhcs/core/steps/function_output_identity.py index f83da45f2f..c606b20dd5 100644 --- a/openhcs/core/steps/function_output_identity.py +++ b/openhcs/core/steps/function_output_identity.py @@ -23,9 +23,9 @@ SourceImageIdentity, SourceImageProvenanceIdentity, ) +from openhcs.core.source_metadata import SourceMetadataFields from openhcs.core.source_matching import ( source_component_metadata_items, - with_source_component_metadata, ) from openhcs.core.source_projection import OpenHCSPlaneAddress from openhcs.microscopes.microscope_interfaces import FilenameParser @@ -144,13 +144,15 @@ def component_metadata( source_metadata: SourceComponentMetadata | None = None, ) -> SourceComponentMetadata: """Return parser-compatible component metadata for this output identity.""" - metadata = dict(source_metadata or {}) - metadata.update(self.component_values) - for component, value in source_component_metadata_items(self.component_values): - metadata = with_source_component_metadata(metadata, component, value) - if self.extension is not None: - metadata["extension"] = self.extension - return metadata + metadata = source_metadata if source_metadata is not None else {} + return SourceMetadataFields.with_fields( + SourceMetadataFields.composition_snapshot(metadata), + self.component_values, + components=source_component_metadata_items(self.component_values), + after_components=( + {"extension": self.extension} if self.extension is not None else None + ), + ) def filename_component_metadata(self) -> SourceComponentMetadata: """Return parser-compatible metadata used to construct the output filename.""" @@ -318,7 +320,9 @@ def _qualified_filename( if qualifier is None: return filename if not filename.endswith(extension): - raise ValueError("Constructed filename does not retain its declared extension.") + raise ValueError( + "Constructed filename does not retain its declared extension." + ) return f"{filename[:-len(extension)]}_{qualifier}{extension}" @staticmethod @@ -584,8 +588,10 @@ def _identity_from_metadata_uncached( identity = cls._identity_from_metadata( metadata.source_component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - metadata.source_component_metadata, metadata.source_path, - parser=parser, identity_cache=identity_cache, + metadata.source_component_metadata, + metadata.source_path, + parser=parser, + identity_cache=identity_cache, ), source="payload component metadata", ) @@ -639,8 +645,10 @@ def _identity_from_metadata_uncached( identity = cls._identity_from_metadata( source_identity.component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - source_identity.component_metadata, source_identity.path, - parser=parser, identity_cache=identity_cache, + source_identity.component_metadata, + source_identity.path, + parser=parser, + identity_cache=identity_cache, ), source="single represented payload source metadata", ) @@ -698,8 +706,10 @@ def filename_identity_from_metadata( identity = cls._identity_from_metadata( metadata.source_component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - metadata.source_component_metadata, metadata.source_path, - parser=parser, identity_cache=identity_cache, + metadata.source_component_metadata, + metadata.source_path, + parser=parser, + identity_cache=identity_cache, ), source="payload component metadata", ) @@ -898,8 +908,10 @@ def _source_stack_source_identity( identity = cls._identity_from_metadata( source_identity.component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - source_identity.component_metadata, source_identity.path, - parser=parser, identity_cache=identity_cache, + source_identity.component_metadata, + source_identity.path, + parser=parser, + identity_cache=identity_cache, ), source=f"represented source identity {identity_index} metadata", ) diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index 35681e5fb9..948d937ec2 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -2084,7 +2084,7 @@ def execution_group_source_payload( if component is None or self.group_key is None: return source_payload metadata = image_payload_metadata(source_payload) - component_metadata = dict(metadata.source_component_metadata or {}) + component_metadata = metadata.source_component_metadata or {} component_metadata = with_source_component_metadata( component_metadata, component, diff --git a/openhcs/core/virtual_workspace_metadata.py b/openhcs/core/virtual_workspace_metadata.py index ff4abc6d0c..ded7fe31be 100644 --- a/openhcs/core/virtual_workspace_metadata.py +++ b/openhcs/core/virtual_workspace_metadata.py @@ -21,9 +21,9 @@ from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.source_bindings import SourceProjectionRole from openhcs.core.source_metadata import ( + DurableSourceMetadata, SourceMetadataMapping, SourceMetadataScalar, - SourceMetadataValue, SourceVoxelSpacing, ) from openhcs.core.source_projection import ( @@ -240,9 +240,12 @@ def update(data): ) subdirectory[FIELDS.IMAGE_FILES] = list(published_paths) subdirectory[FIELDS.MICROSCOPE_HANDLER_NAME] = microscope_handler_name - subdirectory[FIELDS.SOURCE_FILENAME_PARSER_NAME] = source_filename_parser_name + subdirectory[FIELDS.SOURCE_FILENAME_PARSER_NAME] = ( + source_filename_parser_name + ) subdirectory[FIELDS.AVAILABLE_BACKENDS] = { - **subdirectory.get(FIELDS.AVAILABLE_BACKENDS, {}), backend: True + **subdirectory.get(FIELDS.AVAILABLE_BACKENDS, {}), + backend: True, } if is_main: subdirectory[serializer.MAIN_FIELD] = True @@ -688,51 +691,7 @@ def normalize_metadata_fields(metadata_fields: JsonValue) -> SourceMetadataMappi raise RuntimeError( "virtual_workspace source metadata values must be mappings." ) - return MappingProxyType( - { - str( - key - ): VirtualWorkspaceSourceMetadataEntries.normalize_metadata_value(value) - for key, value in metadata_fields.items() - } - ) - - @staticmethod - def normalize_metadata_value(value: JsonValue) -> SourceMetadataValue: - if isinstance(value, Mapping): - return MappingProxyType( - { - str( - nested_key - ): VirtualWorkspaceSourceMetadataEntries.require_scalar_metadata_value( - nested_value - ) - for nested_key, nested_value in value.items() - } - ) - return VirtualWorkspaceSourceMetadataEntries.require_scalar_metadata_value( - value - ) - - @staticmethod - def require_scalar_metadata_value(value: JsonValue) -> SourceMetadataScalar: - # Scalar fast path first: metadata values are overwhelmingly scalars, - # and the container ABC isinstance checks below are comparatively - # expensive per field. - if value is None or isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, Mapping) or ( - isinstance(value, Sequence) and not isinstance(value, str) - ): - raise RuntimeError( - "virtual_workspace source metadata supports scalar values and " - "one-level scalar mappings only." - ) - raise RuntimeError( - "virtual_workspace source metadata scalar values must be strings, " - "numbers, booleans, or null." - ) - return value + return DurableSourceMetadata.from_mapping(metadata_fields) def metadata_for(self, virtual_path: str) -> SourceMetadataMapping: metadata = self.entries.get(virtual_path) diff --git a/openhcs/interop/cellprofiler/analyst_export.py b/openhcs/interop/cellprofiler/analyst_export.py index 7c61aa1e44..4d1df75243 100644 --- a/openhcs/interop/cellprofiler/analyst_export.py +++ b/openhcs/interop/cellprofiler/analyst_export.py @@ -63,7 +63,7 @@ SourceImageProvenance, ) from openhcs.core.source_bindings import CompiledSourceBindingPlan -from openhcs.core.source_metadata import SourceMetadataRoleView, SourceMetadataScalar +from openhcs.core.source_metadata import SourceMetadataFields, SourceMetadataScalar from openhcs.core.source_workspace_projection import ( VirtualWorkspacePathLookup, VirtualWorkspaceSourceProjectionAuthority, @@ -919,7 +919,7 @@ def collect_image_provenance( ] = source_path.as_uri() if component_metadata is not None: metadata_items.update( - SourceMetadataRoleView(component_metadata).original_items() + SourceMetadataFields.original_items(component_metadata) ) image_number = self.image_set_numbering.for_source_slice( scope=scope, diff --git a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py index 3682699839..3c0745c822 100644 --- a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py +++ b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py @@ -58,7 +58,7 @@ source_component_metadata_consensus, ) from openhcs.core.source_metadata import ( - SourceMetadataRoleView, + SourceMetadataFields, ) from openhcs.interop.cellprofiler.image_set_numbering import ( CellProfilerImageSetNumbering, @@ -583,8 +583,7 @@ def _source_metadata_measurement_rows( ).source_component_metadata if metadata is None: continue - role_view = SourceMetadataRoleView(metadata) - original_metadata = dict(role_view.original_items()) + original_metadata = dict(SourceMetadataFields.original_items(metadata)) if original_metadata: rows.append((image_numbers_by_slice[slice_index], original_metadata)) return tuple(rows) diff --git a/tests/integration/test_source_store_code_mode_zmq.py b/tests/integration/test_source_store_code_mode_zmq.py index f8858301d9..749b273b4a 100644 --- a/tests/integration/test_source_store_code_mode_zmq.py +++ b/tests/integration/test_source_store_code_mode_zmq.py @@ -42,7 +42,7 @@ SourceFilterSubject, SourceSelector, ) -from openhcs.core.source_metadata import SourceMetadataRoleView +from openhcs.core.source_metadata import SourceMetadataFields from openhcs.core.steps.function_step import FunctionStep from openhcs.microscopes.bioformats_adapter import ( BioFormatsAdapterUnavailableError, @@ -316,9 +316,7 @@ def test_code_mode_and_zmq_wire_preserve_mixed_store_sources( } for alias, (source_path, pixels) in stores.items(): virtual_path, record = records_by_alias[alias] - filter_paths = SourceMetadataRoleView( - record.source_metadata - ).source_filter_paths() + filter_paths = SourceMetadataFields.source_filter_paths(record.source_metadata) assert source_path.name in filter_paths assert str(source_path) in filter_paths np.testing.assert_array_equal( diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py index 1a2662a51c..5d3660d75c 100644 --- a/tests/unit/test_runtime_source_resolution_snapshot.py +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -11,9 +11,7 @@ from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache from openhcs.core.source_binding_selection import ( DeclaredSourceMetadataRecord, - ResolvedSourceMetadataRecord, SourceBindingMatchedImageSet, - SourceMetadataRecord, SourcePatternResolutionContext, ) from openhcs.core.source_bindings import ( @@ -22,7 +20,10 @@ SourceBindingRuntimeContext, ) from openhcs.core.source_matching import SourceImageSetIdentityPolicy -from openhcs.core.source_metadata import ORIGINAL_SOURCE_METADATA_FIELD +from openhcs.core.source_metadata import ( + ORIGINAL_SOURCE_METADATA_FIELD, + ResolvedSourceMetadataRecord, +) from openhcs.core.source_projection import OpenHCSPlaneAddress, SourcePlaneProjection from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection from openhcs.microscopes.microscope_interfaces import FilenameParseResult @@ -72,7 +73,7 @@ def snapshot(cache, source_projection, parser, rules=()): def test_declared_record_resolves_live_parser_and_nested_metadata(): nested = {"Plate": "before"} - record = SourceMetadataRecord.from_mapping( + record = DeclaredSourceMetadataRecord.from_mapping( {ORIGINAL_SOURCE_METADATA_FIELD: nested, "site": 8} ) assert isinstance(record, DeclaredSourceMetadataRecord) @@ -218,19 +219,19 @@ def test_snapshot_rejects_nested_values_outside_existing_metadata_grammar(): def test_declared_and_resolved_records_preserve_family_value_identity(): - declared = SourceMetadataRecord.from_mapping({"site": 1, "well": "A01"}) - resolved = ResolvedSourceMetadataRecord.from_resolved_mapping( - {"site": 1, "well": "A01"} - ) + declared = DeclaredSourceMetadataRecord.from_mapping({"site": 1, "well": "A01"}) + resolved = ResolvedSourceMetadataRecord.from_mapping({"site": 1, "well": "A01"}) assert declared == resolved assert hash(declared) == hash(resolved) assert hash(declared) == hash((declared.fields,)) - assert declared != SourceMetadataRecord.from_mapping({"site": 2, "well": "A01"}) + assert declared != DeclaredSourceMetadataRecord.from_mapping( + {"site": 2, "well": "A01"} + ) nested = {ORIGINAL_SOURCE_METADATA_FIELD: {"Plate": "A"}} with pytest.raises(TypeError): - hash(SourceMetadataRecord.from_mapping(nested)) + hash(DeclaredSourceMetadataRecord.from_mapping(nested)) with pytest.raises(TypeError): - hash(ResolvedSourceMetadataRecord.from_resolved_mapping(nested)) + hash(ResolvedSourceMetadataRecord.from_mapping(nested)) def test_snapshot_owns_position_and_projection_map_views(): @@ -302,3 +303,30 @@ def test_warmed_cache_transport_reconstructs_all_derived_defaults(context_owned) rebuilt = snapshot(restored, source_projection, parser) assert rebuilt is not context assert rebuilt.metadata_for_path(PATH) == context.metadata_for_path(PATH) + + +def test_durable_workspace_mapping_still_runs_selector_parser_and_rule_fallbacks(): + from openhcs.core.source_metadata import DurableSourceMetadata, SourceMetadataRecord + + metadata = DurableSourceMetadata.from_mapping({"site": 8, "literal": "kept"}) + parser = CountingParser() + context = SourcePatternResolutionContext.from_sources( + parser=parser, + source_paths_by_virtual_path={PATH: "/source/image.tif"}, + source_metadata_by_path={PATH: metadata}, + metadata_rules=( + MetadataExtractionRule( + source=MetadataSource.FILE_NAME, pattern=r"(?PA01)" + ), + ), + ) + assert not isinstance(metadata, SourceMetadataRecord) + assert isinstance( + context.source_metadata_by_path[PATH], DeclaredSourceMetadataRecord + ) + resolved = context.metadata_for_path(PATH) + assert resolved["site"] == 8 + assert resolved["well"] == "A01" + assert resolved["literal"] == "kept" + assert resolved[ORIGINAL_SOURCE_METADATA_FIELD]["Plate"] == "A01" + assert parser.calls == [PATH] diff --git a/tests/unit/test_source_binding_workspace.py b/tests/unit/test_source_binding_workspace.py index 32436c2527..94c7c0c6c5 100644 --- a/tests/unit/test_source_binding_workspace.py +++ b/tests/unit/test_source_binding_workspace.py @@ -12,7 +12,7 @@ from openhcs.core.source_binding_workspace import SourceBindingWorkspaceProjector from openhcs.core.runtime_tabular_values import FieldSpec from openhcs.core.source_metadata import ( - SourceMetadataRoleView, + SourceMetadataFields, SourceVoxelSpacing, SourceVoxelSpacingUnit, SOURCE_VOXEL_SPACING_FIELD, @@ -330,9 +330,7 @@ def test_source_binding_workspace_projector_assigns_selector_channels(tmp_path): } assert all("image_type" not in source for source in serialized_sources) filter_paths_by_alias = { - str(source["source_alias"]): SourceMetadataRoleView( - source - ).source_filter_paths() + str(source["source_alias"]): SourceMetadataFields.source_filter_paths(source) for source in serialized_sources } assert filter_paths_by_alias == { @@ -386,7 +384,7 @@ def test_source_binding_workspace_projects_semantic_identity_after_raw_selection assert projection.address.value_for(AllComponents.CHANNEL) == "MCP_DNA" original_metadata = dict( - SourceMetadataRoleView(projection.source_metadata).original_items() + SourceMetadataFields.original_items(projection.source_metadata) ) assert original_metadata[AllComponents.CHANNEL.value] == "1" @@ -609,7 +607,7 @@ def test_group_address_preserves_distinct_source_well_as_literal_metadata(tmp_pa source_metadata = next(iter(metadata[FIELDS.SOURCE_METADATA].values())) assert source_metadata["well"] == "Sequence1" - assert dict(SourceMetadataRoleView(source_metadata).original_items()) == { + assert dict(SourceMetadataFields.original_items(source_metadata)) == { "Well": "A01", "FrameNumber": "0000", "Run": "Sequence1", @@ -788,7 +786,7 @@ def test_order_source_sets_join_imported_metadata_across_aliases(tmp_path): assert {projection.source_alias for projection in projections} == {"DNA", "Actin"} for projection in projections: original = dict( - SourceMetadataRoleView(projection.source_metadata).original_items() + SourceMetadataFields.original_items(projection.source_metadata) ) assert original["Compound"] == "DMSO" assert original["Dose"] == "0" @@ -912,7 +910,7 @@ def test_metadata_source_sets_propagate_imported_metadata(tmp_path): assert len(projections) == 2 assert all( - dict(SourceMetadataRoleView(item.source_metadata).original_items())["Compound"] + dict(SourceMetadataFields.original_items(item.source_metadata))["Compound"] == "DrugA" for item in projections ) @@ -958,7 +956,7 @@ def test_imported_metadata_duplicate_join_uses_first_matching_row(tmp_path): assert metadata["Site"] == "1" assert metadata["Compound"] == "First" - original = dict(SourceMetadataRoleView(metadata).original_items()) + original = dict(SourceMetadataFields.original_items(metadata)) assert original["Site"] == "1" assert original["Plate"] == "20585" assert original["Well"] == "A01" @@ -1006,7 +1004,7 @@ def test_imported_metadata_later_stage_overrides_extracted_field(tmp_path): assert metadata["Plate"] == "plate_1" assert metadata["Dose"] == "0" assert projection.address.value_for(AllComponents.WELL) == "A01" - original = dict(SourceMetadataRoleView(metadata).original_items()) + original = dict(SourceMetadataFields.original_items(metadata)) assert original["Plate"] == "plate_1" assert original["Well"] == "A01" assert original["ChannelNumber"] == "2" @@ -1058,7 +1056,7 @@ def test_imported_metadata_coerces_join_and_payload_values_through_declared_type filemanager=_filemanager(), ) projection = projection_set.projections[0] - original = dict(SourceMetadataRoleView(projection.source_metadata).original_items()) + original = dict(SourceMetadataFields.original_items(projection.source_metadata)) assert original["Site"] == 5 assert original["Dose"] == 0.25 @@ -1072,7 +1070,7 @@ def test_imported_metadata_coerces_join_and_payload_values_through_declared_type pixel_size=1.0, ) serialized = next(iter(metadata[FIELDS.SOURCE_METADATA].values())) - serialized_original = dict(SourceMetadataRoleView(serialized).original_items()) + serialized_original = dict(SourceMetadataFields.original_items(serialized)) assert serialized_original["Dose"] == 0.25 assert serialized_original["Frame"] == 7 @@ -1161,7 +1159,7 @@ def test_imported_metadata_skips_source_sets_with_partial_join_identity(tmp_path ) assert "Compound" not in dict( - SourceMetadataRoleView(projection.source_metadata).original_items() + SourceMetadataFields.original_items(projection.source_metadata) ) @@ -1285,7 +1283,7 @@ def test_source_bindings_reinitialization_resolves_current_imported_metadata(tmp ) ) assert ( - dict(SourceMetadataRoleView(first_metadata).original_items())["Compound"] + dict(SourceMetadataFields.original_items(first_metadata))["Compound"] == "First" ) @@ -1300,7 +1298,7 @@ def test_source_bindings_reinitialization_resolves_current_imported_metadata(tmp assert metadata_path.read_bytes() != first_payload assert ( - dict(SourceMetadataRoleView(second_metadata).original_items())["Compound"] + dict(SourceMetadataFields.original_items(second_metadata))["Compound"] == "Second" ) diff --git a/tests/unit/test_source_image_provenance_hotpath.py b/tests/unit/test_source_image_provenance_hotpath.py index bf3e167d50..552cdadf8e 100644 --- a/tests/unit/test_source_image_provenance_hotpath.py +++ b/tests/unit/test_source_image_provenance_hotpath.py @@ -14,7 +14,7 @@ def test_empty_explicit_provenance_reuses_authoritative_value() -> None: ) fields = SourceImageProvenanceFields(source_provenance=retained) - fields.absorb_explicit_source_provenance(SourceImageProvenance()) + fields.absorb_explicit_source_provenance((None, None, None, ())) assert fields.source_provenance is retained @@ -30,7 +30,9 @@ def test_populated_explicit_provenance_preserves_merge_precedence() -> None: ) fields = SourceImageProvenanceFields(source_provenance=fallback) - fields.absorb_explicit_source_provenance(explicit) + fields.absorb_explicit_source_provenance( + (None, {"channel": "2"}, None, ("Explicit",)) + ) assert fields.source_provenance == SourceImageProvenance( source_path="/input/A01_s001_w1.tif", diff --git a/tests/unit/test_source_metadata_live_mutation.py b/tests/unit/test_source_metadata_live_mutation.py index 19cf5086cd..935fc63373 100644 --- a/tests/unit/test_source_metadata_live_mutation.py +++ b/tests/unit/test_source_metadata_live_mutation.py @@ -3,7 +3,7 @@ import pytest from openhcs.constants.constants import AllComponents -from openhcs.core.source_binding_selection import SourceMetadataRecord +from openhcs.core.source_binding_selection import DeclaredSourceMetadataRecord from openhcs.core.source_matching import ( ORIGINAL_SOURCE_METADATA_FIELD, semantic_source_metadata_value, @@ -31,7 +31,7 @@ def test_metadata_queries_observe_scalar_mutation(readonly_outer): @pytest.mark.parametrize( - "view", (dict, MappingProxyType, SourceMetadataRecord.from_mapping) + "view", (dict, MappingProxyType, DeclaredSourceMetadataRecord.from_mapping) ) def test_literal_queries_observe_nested_original_mutation(view): original = {"Well": "LiteralA"} diff --git a/tests/unit/test_source_metadata_owner.py b/tests/unit/test_source_metadata_owner.py new file mode 100644 index 0000000000..451d5cbb7a --- /dev/null +++ b/tests/unit/test_source_metadata_owner.py @@ -0,0 +1,490 @@ +"""Owned metadata lifetime and its production provenance consumers.""" + +import pickle +from collections.abc import Mapping + +import cloudpickle +from types import MappingProxyType + +import numpy as np +import pytest +from polystore.virtual_workspace import SourcePixelRef + +from openhcs.constants.constants import AllComponents +from openhcs.core.artifacts import ImageArtifactType +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_metadata, +) +from openhcs.core.source_binding_selection import DeclaredSourceMetadataRecord +from openhcs.core.source_bindings import ( + SOURCE_BINDING_ALIAS_METADATA_FIELD, + SourceBindingRuntimeContext, +) +from openhcs.core.source_image_provenance import ( + SourceImageIdentity, + SourceImageProvenance, + SourceImageProvenancePlaneRecord, + SourcePlaneIndexedMetadata, + source_component_metadata_consensus, +) +from openhcs.core.source_matching import ( + source_component_metadata_items, + source_component_metadata_value, + source_component_metadata_values, + source_metadata_value, + with_source_component_metadata, +) +from openhcs.core.source_metadata import ( + ORIGINAL_SOURCE_METADATA_FIELD, + DurableSourceMetadata, + ResolvedSourceMetadataRecord, + SourceMetadataRecord, + SourceMetadataFields, +) +from openhcs.core.source_projection import OpenHCSPlaneAddress, SourcePlaneProjection +from openhcs.core.steps.function_output_identity import FunctionOutputIdentity +from openhcs.core.source_workspace_projection import ( + VirtualWorkspaceImagePayloadProjection, + VirtualWorkspaceSourceProjection, +) +from openhcs.core.virtual_workspace_metadata import ( + VirtualWorkspaceSourceMetadataEntries, +) +from openhcs.serialization.json import to_jsonable + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +def test_owned_snapshot_detaches_nested_fields_and_preserves_record_contract(owner): + nested = {"Well": "literal"} + record = owner.from_mapping({"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: nested}) + nested["Well"] = "changed" + assert source_metadata_value(record, "Well") == "literal" + with pytest.raises(TypeError): + record[ORIGINAL_SOURCE_METADATA_FIELD]["Well"] = "changed" + with pytest.raises(TypeError): + hash(record) + declared = DeclaredSourceMetadataRecord.from_mapping( + {"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: {"Well": "literal"}} + ) + reversed_record = owner.from_mapping( + {ORIGINAL_SOURCE_METADATA_FIELD: {"Well": "literal"}, "well": "A01"} + ) + if isinstance(record, SourceMetadataRecord): + assert record == declared + assert record != reversed_record + else: + assert record != declared and declared != record + assert record == reversed_record + assert record == dict(declared) + + +def test_owned_lookup_keeps_first_duplicate_field_and_full_ordered_record(): + record = ResolvedSourceMetadataRecord((("well", "A01"), ("well", "A02"))) + assert record["well"] == "A01" + assert tuple(record.items()) == (("well", "A01"), ("well", "A01")) + assert record.fields == (("well", "A01"), ("well", "A02")) + assert dict(record) == {"well": "A01"} + assert record != ResolvedSourceMetadataRecord((("well", "A01"),)) + + +def test_runtime_birth_canonicalizes_but_durable_and_derived_values_keep_literal_spelling( + tmp_path, +): + spelling = str(tmp_path / "unused" / ".." / "image.tif") + durable = DurableSourceMetadata.from_mapping({"path": spelling}) + runtime = ResolvedSourceMetadataRecord.from_mapping(durable) + assert durable["path"] == spelling + assert runtime["path"] == str(tmp_path / "image.tif") + derived = SourceMetadataFields.with_fields(runtime, {"new_path": spelling}) + assert isinstance(derived, DurableSourceMetadata) + assert derived["path"] == str(tmp_path / "image.tif") + assert derived["new_path"] == spelling + projection = SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values( + well="A01", site="1", channel="1", z_index="1", timepoint="1" + ), + ref=SourcePixelRef("disk", "plane.tif"), + source_metadata=derived, + ) + assert projection.source_metadata["new_path"] == str(tmp_path / "image.tif") + + +def test_owned_scalar_identity_reuses_literal_metadata_but_not_mutable_carriers(): + fields = {"well": "A01", "channel": "1", "extension": ".tif"} + owner = DurableSourceMetadata.from_mapping(fields) + original = ImagePayloadMetadata( + source_provenance=SourceImageProvenance( + source_path="/input/source.tif", + source_component_metadata=owner, + source_image_names=("DNA",), + ) + ) + first = original.replace_fields() + second = original.with_source_context_from(original) + assert first.source_provenance is not original.source_provenance + assert ( + first.source_provenance.source_identity + is not original.source_provenance.source_identity + ) + assert first.source_component_metadata is owner + assert second.source_component_metadata is owner + assert ( + first.source_provenance.equality_identity + == original.source_provenance.equality_identity + ) + assert to_jsonable(first) == to_jsonable(original) + first.source_provenance.source_identity.path = "/changed.tif" + assert original.source_path == "/input/source.tif" + assert second.source_path == "/input/source.tif" + + +def test_workspace_alias_projection_and_artifact_naming_preserve_owned_metadata(): + source = VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields( + {"well": "A01", "channel": "1", SOURCE_BINDING_ALIAS_METADATA_FIELD: "DNA"} + ) + payload = ImagePayloadMetadata().payload_with(np.ones((2, 2), dtype=np.float32)) + projected = VirtualWorkspaceImagePayloadProjection( + source_metadata=source, source_alias="DNA" + ).apply(payload) + metadata = image_payload_metadata(projected) + assert isinstance(metadata.source_component_metadata, DurableSourceMetadata) + assert SOURCE_BINDING_ALIAS_METADATA_FIELD not in metadata.source_component_metadata + named = ImageArtifactType.normalize_runtime_payload("DerivedDNA", projected) + named_metadata = image_payload_metadata(named) + assert ( + named_metadata.source_component_metadata is metadata.source_component_metadata + ) + assert named_metadata.source_image_names == ("DerivedDNA",) + assert metadata.source_image_names == ("DNA",) + assert ( + named_metadata.source_provenance.source_image_provenance_planes.represented_source_image_names + == ("DNA",) + ) + + +def test_owned_updates_replace_aliases_and_keep_original_literals(): + owner = DurableSourceMetadata.from_mapping( + { + "channel": "1", + "ChannelNumber": "2", + ORIGINAL_SOURCE_METADATA_FIELD: {"ChannelNumber": "literal"}, + } + ) + updated = with_source_component_metadata(owner, AllComponents.CHANNEL, "3") + assert isinstance(updated, DurableSourceMetadata) + assert "ChannelNumber" not in updated + assert source_metadata_value(updated, "ChannelNumber") == "literal" + assert source_component_metadata_values(updated, AllComponents.CHANNEL) == ("3",) + assert owner["channel"] == "1" + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +def test_custom_scalar_remains_live_for_queries_and_new_fingerprints(owner): + class MutableInteger(int): + def __str__(self): + return self.tag + + def __repr__(self): + return self.tag + + value = MutableInteger(1) + value.tag = "before" + record = owner.from_mapping({"channel": value}) + first = SourceImageIdentity(component_metadata=record) + assert source_component_metadata_value(record, AllComponents.CHANNEL) == "before" + value.tag = "after" + assert source_component_metadata_value(record, AllComponents.CHANNEL) == "after" + assert SourceImageIdentity(component_metadata=record).identity != first.identity + assert first.identity[1] == (("channel", "before"),) + + +def test_lazy_role_errors_are_unchanged_by_owned_construction(): + record = DurableSourceMetadata.from_mapping( + { + "channel": "1", + "wellrow": "A", + "wellcolumn": "bad", + ORIGINAL_SOURCE_METADATA_FIELD: 7, + } + ) + assert source_component_metadata_value(record, AllComponents.CHANNEL) == "1" + with pytest.raises(ValueError): + source_component_metadata_value(record, AllComponents.WELL) + with pytest.raises(RuntimeError, match="must be a mapping"): + source_metadata_value(record, "channel") + with pytest.raises(RuntimeError, match="must be a mapping"): + source_component_metadata_values(record, AllComponents.CHANNEL) + + +@pytest.mark.parametrize("invalid", ([1], {"deep": {"unsupported": 1}}, object())) +def test_durable_metadata_keeps_existing_invalid_value_domain(invalid): + with pytest.raises(RuntimeError): + VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields( + {"field": invalid} + ) + + +def test_source_identity_mapping_equality_keeps_original_class_and_current_field_boundaries(): + owner = DurableSourceMetadata.from_mapping({"flag": True}) + identity = SourceImageIdentity("source.tif", owner) + raw = SourceImageIdentity("source.tif", MappingProxyType({"flag": True})) + assert identity == raw and raw == identity + assert identity != SourceImageIdentity("source.tif", {"flag": 1}) + assert SourceImageIdentity(component_metadata=None) != SourceImageIdentity( + component_metadata={} + ) + with pytest.raises(TypeError): + hash(identity) + + class Child(SourceImageIdentity): + pass + + assert identity.__eq__(Child("source.tif", owner)) is NotImplemented + + class ReportedIdentity: + @property + def __class__(self): + return SourceImageIdentity + + def __getattribute__(self, name): + if name in ("path", "component_metadata", "_identity"): + reads.append(name) + return getattr(raw, name) + return object.__getattribute__(self, name) + + reads = [] + assert identity.__eq__(ReportedIdentity()) is True + assert reads == ["path", "component_metadata", "_identity"] + + nested = {"literal": "before"} + left = SourceImageIdentity( + component_metadata={ORIGINAL_SOURCE_METADATA_FIELD: nested} + ) + right = SourceImageIdentity( + component_metadata={ORIGINAL_SOURCE_METADATA_FIELD: {"literal": "before"}} + ) + captured = left.identity + assert left == right + nested["literal"] = "after" + assert left != right and left.identity == captured + + +def test_raw_composition_snapshots_once_before_consensus_reads(): + class ObservableMapping(Mapping): + def __init__(self, values): + self.values = values + self.key_reads = self.value_reads = 0 + + def __iter__(self): + self.key_reads += 1 + return iter(self.values) + + def __len__(self): + return len(self.values) + + def __getitem__(self, key): + self.value_reads += 1 + return self.values[key] + + raw = ObservableMapping({"well": "A01"}) + consensus = source_component_metadata_consensus((raw, raw)) + assert dict(consensus) == {"well": "A01"} + assert raw.key_reads == 2 and raw.value_reads == 2 + snapshot = SourceMetadataFields.composition_snapshot(raw) + raw.values["well"] = "A02" + assert snapshot["well"] == "A01" + assert source_component_metadata_consensus((raw, raw))["well"] == "A02" + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +@pytest.mark.parametrize("serializer", (pickle, cloudpickle)) +def test_owned_transport_stores_only_authoritative_fields_and_rebuilds_local_views( + owner, serializer +): + record = owner.from_mapping( + {"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: {"literal": "value"}} + ) + before = SourceImageIdentity(component_metadata=record).identity + assert source_metadata_value(record, "literal") == "value" + assert source_component_metadata_value(record, AllComponents.WELL) == "A01" + assert record._views + restored = serializer.loads(serializer.dumps(record)) + assert isinstance(restored, owner) + assert restored.fields == record.fields + assert not restored._views + assert restored.metadata_contents is not record.metadata_contents + assert to_jsonable(restored) == to_jsonable(record) + assert SourceImageIdentity(component_metadata=restored).identity == before + with pytest.raises(TypeError): + restored[ORIGINAL_SOURCE_METADATA_FIELD]["literal"] = "changed" + + +@pytest.mark.parametrize("owner", (ResolvedSourceMetadataRecord, DurableSourceMetadata)) +def test_transport_does_not_recanonicalize_stored_absolute_values_or_validate_unused_roles( + owner, tmp_path +): + directory = tmp_path / "original" + directory.mkdir() + spelling = str(directory / "image.tif") + record = owner.from_mapping({"path": spelling, ORIGINAL_SOURCE_METADATA_FIELD: 7}) + before = SourceImageIdentity(component_metadata=record).identity + directory.rmdir() + target = tmp_path / "replacement" + target.mkdir() + directory.symlink_to(target, target_is_directory=True) + assert str((directory / "image.tif").resolve()) != spelling + restored = pickle.loads(pickle.dumps(record)) + assert restored["path"] == spelling + assert SourceImageIdentity(component_metadata=restored).identity == before + with pytest.raises(RuntimeError, match="must be a mapping"): + source_metadata_value(restored, "unused") + + +def test_ordered_component_batch_matches_sequential_raw_updates(): + fields = { + "Well": "old", + "Site": "old", + "ChannelNumber": "old", + "ZIndex": "old", + "Timepoint": "old", + ORIGINAL_SOURCE_METADATA_FIELD: {"Well": "literal"}, + } + components = tuple( + (component, str(index)) for index, component in enumerate(AllComponents, 1) + ) + raw = dict(fields) + for component, value in components: + raw = with_source_component_metadata(raw, component, value) + record = DurableSourceMetadata.from_mapping(fields) + updated = SourceMetadataFields.with_fields(record, {}, components=components) + assert isinstance(updated, DurableSourceMetadata) + assert to_jsonable(updated) == to_jsonable(raw) + assert tuple(updated) == tuple(raw) + assert source_metadata_value(updated, "Well") == "literal" + + +@pytest.mark.parametrize("existing_extension", (False, True)) +def test_output_identity_batch_keeps_original_field_order_and_extension_precedence( + existing_extension, +): + fields = {"literal": "kept", "Well": "old"} + if existing_extension: + fields["extension"] = ".old" + identity = FunctionOutputIdentity( + component_values={"well": "A01", "channel": "1"}, + extension=".tif", + source="control", + ) + expected = dict(fields) + expected.update(identity.component_values) + for component, value in source_component_metadata_items(identity.component_values): + expected = with_source_component_metadata(expected, component, value) + expected["extension"] = ".tif" + actual = identity.component_metadata(DurableSourceMetadata.from_mapping(fields)) + assert tuple(actual.items()) == tuple(expected.items()) + assert to_jsonable(actual) == expected + + +def test_duplicate_record_keys_collapse_at_source_identity_mapping_boundary(): + record = ResolvedSourceMetadataRecord((("well", "A01"), ("well", "A02"))) + expected = SourceImageIdentity(component_metadata=dict(record)) + actual = SourceImageIdentity(component_metadata=record) + assert actual == expected + assert actual.identity == expected.identity + assert tuple(actual.component_metadata.items()) == (("well", "A01"),) + assert record.fields == (("well", "A01"), ("well", "A02")) + + +def test_stringified_key_collisions_keep_platform_map_last_value_at_owned_birth(): + fields = {1: "first", "1": "last"} + durable = VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields(fields) + assert tuple(durable.items()) == (("1", "last"),) + projection = SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values( + well="A01", site="1", channel="1", z_index="1", timepoint="1" + ), + ref=SourcePixelRef("disk", "plane.tif"), + source_metadata=fields, + ) + assert tuple(projection.source_metadata.items()) == (("1", "last"),) + declaration = DeclaredSourceMetadataRecord.from_mapping(fields) + assert declaration.fields == (("1", "first"), ("1", "last")) + assert declaration["1"] == "first" + + +@pytest.mark.parametrize( + "decoder", + ( + ResolvedSourceMetadataRecord.from_mapping, + DurableSourceMetadata.from_mapping, + VirtualWorkspaceSourceMetadataEntries.normalize_metadata_fields, + ), +) +def test_owned_map_birth_validates_overwritten_stringified_key_values(decoder): + with pytest.raises((TypeError, RuntimeError)): + decoder({1: [], "1": "valid"}) + + +def test_mapping_and_ordered_record_equality_namespaces_stay_separate(): + ordered = ResolvedSourceMetadataRecord.from_mapping({"well": "A01", "site": "1"}) + reordered = ResolvedSourceMetadataRecord.from_mapping({"site": "1", "well": "A01"}) + mapping = DurableSourceMetadata.from_mapping(ordered) + reordered_mapping = DurableSourceMetadata.from_mapping(reordered) + assert ordered != reordered + assert mapping == reordered_mapping + assert mapping == dict(ordered) and dict(ordered) == mapping + for record in ( + ordered, + reordered, + DeclaredSourceMetadataRecord.from_mapping(dict(ordered)), + ): + assert mapping != record and record != mapping + assert hash(ordered) == hash( + DeclaredSourceMetadataRecord.from_mapping(dict(ordered)) + ) + with pytest.raises(TypeError): + hash(mapping) + + +@pytest.mark.parametrize( + "factory", + ( + lambda metadata: VirtualWorkspaceSourceMetadataEntries({"plane.tif": metadata}), + lambda metadata: VirtualWorkspaceImagePayloadProjection( + source_metadata=metadata + ), + lambda metadata: VirtualWorkspaceSourceProjection({}, {"plane.tif": metadata}), + lambda metadata: SourceImageProvenancePlaneRecord(component_metadata=metadata), + lambda metadata: SourcePlaneIndexedMetadata(metadata, 0, 1), + lambda metadata: SourceBindingRuntimeContext( + source_metadata_by_path={"plane.tif": metadata} + ), + lambda metadata: SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values( + well="A01", site="1", channel="1", z_index="1", timepoint="1" + ), + ref=SourcePixelRef("disk", "plane.tif"), + source_metadata=metadata, + ), + ), +) +def test_public_mapping_field_carriers_keep_content_equality(factory): + owner = DurableSourceMetadata.from_mapping({"well": "A01", "site": "1"}) + reordered = DurableSourceMetadata.from_mapping({"site": "1", "well": "A01"}) + raw = MappingProxyType({"well": "A01", "site": "1"}) + assert factory(owner) == factory(reordered) + assert factory(owner) == factory(raw) + assert factory(raw) == factory(owner) + with pytest.raises(TypeError): + hash(factory(owner)) + + +def test_durable_tuple_birth_uses_mapping_last_value_policy(): + metadata = DurableSourceMetadata((("well", "A01"), ("well", "A02"))) + assert metadata.fields == (("well", "A02"),) + assert metadata == {"well": "A02"} + assert ( + SourceImageIdentity(component_metadata=metadata).identity + == SourceImageIdentity(component_metadata={"well": "A02"}).identity + ) diff --git a/tests/unit/test_source_projection.py b/tests/unit/test_source_projection.py index a2a590d50a..5871a9745f 100644 --- a/tests/unit/test_source_projection.py +++ b/tests/unit/test_source_projection.py @@ -11,7 +11,7 @@ from openhcs.core.source_bindings import SourceProjectionRole from openhcs.core.source_metadata import ( ORIGINAL_SOURCE_METADATA_FIELD, - SourceMetadataRoleView, + SourceMetadataFields, ) from openhcs.core.source_projection import ( OpenHCSPlaneAddress, @@ -269,7 +269,7 @@ def test_source_projection_preserves_provenance_owned_component_remaps( assert source_metadata[component.value] == address_value assert ( - dict(SourceMetadataRoleView(source_metadata).original_items())[component.value] + dict(SourceMetadataFields.original_items(source_metadata))[component.value] == source_value ) From d2ae15940246ca0fe143c5300ea4b1f3a395f2ec Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 19:34:14 -0400 Subject: [PATCH 025/125] refactor(metadata): derive nullable scalar grammar from primitive contract --- openhcs/core/source_metadata.py | 7 +- tests/unit/test_source_metadata_owner.py | 119 ++++++++++++++++++++++- 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/openhcs/core/source_metadata.py b/openhcs/core/source_metadata.py index cdc7d47641..2b819bb0e9 100644 --- a/openhcs/core/source_metadata.py +++ b/openhcs/core/source_metadata.py @@ -30,7 +30,8 @@ SOURCE_VOXEL_SPACING_FIELD = "OpenHCSSourceVoxelSpacingZYX" SOURCE_VOXEL_SPACING_UNIT_FIELD = "OpenHCSSourceVoxelSpacingUnit" -SourceMetadataScalar: TypeAlias = str | int | float | bool | None +SourceMetadataNonNullScalar: TypeAlias = str | int | float | bool +SourceMetadataScalar: TypeAlias = SourceMetadataNonNullScalar | None SourceMetadataValue: TypeAlias = ( SourceMetadataScalar | Mapping[str, SourceMetadataScalar] ) @@ -499,7 +500,7 @@ def __eq__(self, other: object) -> bool: @staticmethod def normalized_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: - if value is None or isinstance(value, (str, int, float, bool)): + if value is None or isinstance(value, SourceMetadataNonNullScalar): return value if isinstance(value, Mapping) or ( isinstance(value, Sequence) and not isinstance(value, str) @@ -547,7 +548,7 @@ def source_metadata_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: if value is None: return None - if not isinstance(value, (str, int, float, bool)): + if not isinstance(value, SourceMetadataNonNullScalar): raise TypeError( "Source metadata scalar values must be str, int, float, bool, or None, " f"got {type(value).__name__}." diff --git a/tests/unit/test_source_metadata_owner.py b/tests/unit/test_source_metadata_owner.py index 451d5cbb7a..a092129dfd 100644 --- a/tests/unit/test_source_metadata_owner.py +++ b/tests/unit/test_source_metadata_owner.py @@ -1,7 +1,7 @@ """Owned metadata lifetime and its production provenance consumers.""" import pickle -from collections.abc import Mapping +from collections.abc import Mapping, Sequence import cloudpickle from types import MappingProxyType @@ -41,6 +41,7 @@ ResolvedSourceMetadataRecord, SourceMetadataRecord, SourceMetadataFields, + source_metadata_scalar, ) from openhcs.core.source_projection import OpenHCSPlaneAddress, SourcePlaneProjection from openhcs.core.steps.function_output_identity import FunctionOutputIdentity @@ -488,3 +489,119 @@ def test_durable_tuple_birth_uses_mapping_last_value_policy(): SourceImageIdentity(component_metadata=metadata).identity == SourceImageIdentity(component_metadata={"well": "A02"}).identity ) + + +def test_scalar_admission_preserves_primitives_subclasses_and_scalar_container_precedence(): + class Integer(int): + pass + + class Real(float): + pass + + class TextMapping(str, Mapping): + pass + + values = ( + None, + False, + True, + 0, + -3, + 1.25, + "literal", + Integer(7), + Real(2.5), + TextMapping("literal"), + ) + for value in values: + assert DurableSourceMetadata.normalized_scalar(value) is value + assert source_metadata_scalar(value) is value + assert isinstance(values[-1], Mapping) + assert isinstance(values[-1], Sequence) + + +def test_durable_scalar_admission_preserves_rejected_container_and_scalar_errors(): + class MappingSequence(Mapping, Sequence): + def __getitem__(self, key): + raise KeyError(key) + + def __iter__(self): + return iter(()) + + def __len__(self): + return 0 + + container_error = ( + "virtual_workspace source metadata supports scalar values and " + "one-level scalar mappings only." + ) + scalar_error = ( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + for value in ( + {}, + [], + (), + range(1), + b"bytes", + bytearray(b"bytes"), + MappingSequence(), + ): + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(value) + assert str(caught.value) == container_error + for value in (set(), frozenset(), 1j, object()): + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(value) + assert str(caught.value) == scalar_error + + +def test_scalar_admission_keeps_none_identity_and_reported_class_read_order(): + class ReportedClass: + def __init__(self, reports): + self.reports = reports + self.reads = [] + + @property + def __class__(self): + reported = self.reports[min(len(self.reads), len(self.reports) - 1)] + self.reads.append(reported) + return reported + + reported_none = ReportedClass((type(None),)) + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(reported_none) + assert str(caught.value) == ( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + assert reported_none.reads == [type(None)] * 6 + + reported_none = ReportedClass((type(None),)) + with pytest.raises(TypeError, match="Source metadata scalar values must be"): + source_metadata_scalar(reported_none) + assert reported_none.reads == [type(None)] * 4 + + reported_integer = ReportedClass((object, int, object)) + assert DurableSourceMetadata.normalized_scalar(reported_integer) is reported_integer + assert reported_integer.reads == [object, int] + + changing_class = ReportedClass( + (object, object, object, object, object, Sequence, str) + ) + with pytest.raises(RuntimeError) as caught: + DurableSourceMetadata.normalized_scalar(changing_class) + assert str(caught.value) == ( + "virtual_workspace source metadata scalar values must be strings, " + "numbers, booleans, or null." + ) + assert changing_class.reads == [ + object, + object, + object, + object, + object, + Sequence, + str, + ] From fbb7907c6c5ca7e80179bfffaf03e06bf951644a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 19:43:40 -0400 Subject: [PATCH 026/125] chore: drop unrelated output identity formatting from parked metadata work --- .../core/steps/function_output_identity.py | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/openhcs/core/steps/function_output_identity.py b/openhcs/core/steps/function_output_identity.py index c606b20dd5..a9775369ec 100644 --- a/openhcs/core/steps/function_output_identity.py +++ b/openhcs/core/steps/function_output_identity.py @@ -320,9 +320,7 @@ def _qualified_filename( if qualifier is None: return filename if not filename.endswith(extension): - raise ValueError( - "Constructed filename does not retain its declared extension." - ) + raise ValueError("Constructed filename does not retain its declared extension.") return f"{filename[:-len(extension)]}_{qualifier}{extension}" @staticmethod @@ -588,10 +586,8 @@ def _identity_from_metadata_uncached( identity = cls._identity_from_metadata( metadata.source_component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - metadata.source_component_metadata, - metadata.source_path, - parser=parser, - identity_cache=identity_cache, + metadata.source_component_metadata, metadata.source_path, + parser=parser, identity_cache=identity_cache, ), source="payload component metadata", ) @@ -645,10 +641,8 @@ def _identity_from_metadata_uncached( identity = cls._identity_from_metadata( source_identity.component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - source_identity.component_metadata, - source_identity.path, - parser=parser, - identity_cache=identity_cache, + source_identity.component_metadata, source_identity.path, + parser=parser, identity_cache=identity_cache, ), source="single represented payload source metadata", ) @@ -706,10 +700,8 @@ def filename_identity_from_metadata( identity = cls._identity_from_metadata( metadata.source_component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - metadata.source_component_metadata, - metadata.source_path, - parser=parser, - identity_cache=identity_cache, + metadata.source_component_metadata, metadata.source_path, + parser=parser, identity_cache=identity_cache, ), source="payload component metadata", ) @@ -908,10 +900,8 @@ def _source_stack_source_identity( identity = cls._identity_from_metadata( source_identity.component_metadata, extension=FunctionOutputExtensionAuthority.from_source( - source_identity.component_metadata, - source_identity.path, - parser=parser, - identity_cache=identity_cache, + source_identity.component_metadata, source_identity.path, + parser=parser, identity_cache=identity_cache, ), source=f"represented source identity {identity_index} metadata", ) From 1087c9330f368b6d6b9928b97f0eb6d9d5c85d2f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 21:03:33 -0400 Subject: [PATCH 027/125] Document measured payoff bounds and rejected standalone routes for #384 --- .../shared_runtime_plumbing_384_20261001.rst | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index d1ca5d920c..ae18505207 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -331,3 +331,53 @@ Pipeline clocks exclude ZMQ server startup, mandatory registry/kernel prewarming and shutdown. Timed runs use the ordinary public OUTCOMES path and default memory observer, without profiling injection. Candidate and baseline must have frozen clean source revisions and the same dependencies, affinity and environment. + +Global lifetime counterfactuals: rejected routes +---------------------------------------------- + +The next ordinary diagnostics and exact saved-input replays reject several +insufficient standalone routes before further implementation. These are payoff +bounds, not candidate speedups. Their observations remain revision-pinned; they +do not supersede the mixed ordinary ABBA above. + +* The eight complete saved IFC geometry owners take 0.4745 seconds in aggregate + median replay; the actual complete spreadsheet renderer takes 1.3662 seconds. + Even eliminating both separate phases has an optimistic ceiling below two + seconds. The isolated geometry/export route is stopped. +* On main 18317499be069fbe58387137f326ac7b6ff57988, all 634 original Advanced + measurement queries take 0.6422 seconds, while complete CPA collection takes + 1.1943 seconds. Fabricated missing-cell counts alone do not admit this route: + eliminating both whole phases still cannot supply two seconds. The saved + query capture is a bounded subset, not a substitute for the complete clock. +* On that same main revision, all 32 3D stack loads and all saves take 0.9927 + and 0.9526 exclusive diagnostic seconds respectively. Eighteen loads select + an actual previously stored named producer; all eighteen allocate independent + main-flow pixel buffers. The full saved step-14 Resize input matches the + stored Monolayer pixels exactly, but differs in filename-derived extension + and per-plane source-name metadata. Direct value reuse would violate existing + buffer isolation and provenance. A narrow named-handoff migration is stopped. +* A low-overhead aggregate diagnostic on that revision measures all 40,673 + ``ImagePayloadMetadata.__post_init__`` calls at 0.8919 inclusive seconds and + all 147,354 ``SourceImageIdentity.__post_init__`` calls at 0.5544 inclusive + seconds. These totals overlap and must never be added. Constructor count + contraction alone cannot justify the multi-second target, even though counts + identify repeated work. The proposed 44.6 percent birth reduction is not a + measured speedup. The wider field-derivation lifetime remains unqualified. + +Every diagnostic uses the ordinary OUTCOMES path, one worker/thread and the +default memory observer. Hook overhead is retained; server startup, prewarming, +compilation and shutdown are outside execution. Exact original-method timing +does not establish that the whole measured duration can be removed. Controllers, +source/dependency freezes, successful outputs and failed earlier captures are +retained under ``/var/tmp`` and the external benchmark evidence directory. + +Evidence: ``measurement-lifetime-actual-input-baseline-replay-20261001/receipt.json``, +``global-plumbing-three-workload-coarse-lifetime-20261001.json``, +``runtime-value-handoff-step14-fixture-metadata-inspection-20261001.json``, and +``metadata-initialization-aggregate-decision-20261001.json``. The full query and +handoff observations reside in their corresponding ``/var/tmp`` captures. + +Main 9a04107492ad90233394cf17524d0ca8e74062bb is normally integrated, including +the merged native-module source-discovery repair in PR416 and source projection +capability repair in PR418. Earlier behavior/source/performance gates retain +their exact source pins. PR394 remains a draft formally closing issue384. From e32b402af17a9baad3c51074e28dbcdd5cf23aa7 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 21:12:52 -0400 Subject: [PATCH 028/125] Record complete field-derivation counterfactual scope for #384 --- .../shared_runtime_plumbing_384_20261001.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index ae18505207..eae4701dee 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -381,3 +381,18 @@ Main 9a04107492ad90233394cf17524d0ca8e74062bb is normally integrated, including the merged native-module source-discovery repair in PR416 and source projection capability repair in PR418. Earlier behavior/source/performance gates retain their exact source pins. PR394 remains a draft formally closing issue384. + +The next bounded union-depth diagnostic on that main measures 3.6594 seconds +across selected metadata construction, provenance projection, component/literal +field lookup and merges. Its 972,354 selected calls enter 102,445 outer selected +regions; nested work is timed once and the final depth is zero. It retains hook +overhead, includes metadata inside callable wrappers and excludes physical +processing, I/O, complete writer preparation and publication ancestors. Thus +this is a plausible wider counterfactual scope, not 3.6594 recoverable seconds. +Reflection controls compare original and wrapped constructors: hints and errors +agree, including the original generated image-metadata constructor's existing +``NameError('InitVar')``. Actual provenance mapping decode succeeds through the +hooks. The next gate is an ordinary production comparison against the existing +nominal immutable-field owner, before further implementation or broad gates. +Evidence: ``metadata-lifetime-union-production-counterfactual-admission-20261001.json`` +and ``metadata-lifetime-union-reflection-pair-control-20261001.json``. From bffa1535ad440d4a0803b369727d737d16b6fce1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 21:41:04 -0400 Subject: [PATCH 029/125] Preserve metadata projection effect and validation order for #384 --- openhcs/core/runtime_image_values.py | 117 ++++++------- openhcs/core/source_image_provenance.py | 16 +- openhcs/core/source_metadata.py | 12 +- .../test_image_payload_projection_algebra.py | 163 ++++++++++++++++++ 4 files changed, 235 insertions(+), 73 deletions(-) diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index 9e9f03e92e..5963e79799 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -239,17 +239,33 @@ def for_array_payload( def __post_init__(self, *source_provenance_values: object) -> None: self.absorb_explicit_source_provenance(source_provenance_values) - self.source_voxel_spacing = self.source_voxel_spacing.with_missing_from( - SourceVoxelSpacing.from_source_metadata(self.source_component_metadata) + for name, value in self.normalized_fields({}).items(): + setattr(self, name, value) + + def normalized_fields(self, changes: Mapping[str, Any]) -> dict[str, Any]: + """Normalize constructor instructions in their source-context effect order.""" + provenance = changes.get("source_provenance", self.source_provenance) + spacing = changes.get("source_voxel_spacing", self.source_voxel_spacing) + spacing = spacing.with_missing_from( + SourceVoxelSpacing.from_source_metadata( + provenance.source_component_metadata + ) ) - self.normalize_source_provenance_fields() - self.normalize_source_spatial_domain_fields() - self.normalize_source_voxel_spacing_fields() + provenance = SourceImageProvenance.normalized(provenance) + domain = changes.get( + "source_spatial_domain", self.source_spatial_domain + ).normalized() + spacing = SourceVoxelSpacing.coerce(spacing) self.validate_source_channel_axis() - if self.plane_axis is not None: - self.plane_axis = RuntimePlaneAxis( - self.plane_axis, - ) + plane_axis = changes.get("plane_axis", self.plane_axis) + if plane_axis is not None: + plane_axis = RuntimePlaneAxis(plane_axis) + return dict( + source_provenance=provenance, + source_voxel_spacing=spacing, + source_spatial_domain=domain, + plane_axis=plane_axis, + ) def validate_source_channel_axis(self) -> None: """Validate the authored channel declaration before transforming axes.""" @@ -261,12 +277,6 @@ def validate_source_channel_axis(self) -> None: "ImagePayloadMetadata.source_channel_axis must be int or None." ) - def require_leading_plane_axis(self, message: str) -> RuntimePlaneAxis: - """Validate a required plane declaration at its metadata authority.""" - if self.plane_axis is None: - raise ValueError(message) - return RuntimePlaneAxis(self.plane_axis) - @property def has_values(self) -> bool: """Return whether this metadata carries any semantic image facts.""" @@ -1413,22 +1423,12 @@ class ImageMetadataProjection(ABC): def project(self) -> ImagePayloadMetadata: return self.metadata.replace_fields(**self.projected_fields()) - @property - def projected_source_provenance(self) -> SourceImageProvenance: - return self.metadata.source_provenance - - @property - def projected_unit_interval_intensity( - self, - ) -> ImageUnitIntervalIntensityMetadata | None: - return self.metadata.unit_interval_intensity - @abstractmethod def projected_fields(self) -> dict[str, Any]: """Derive constructor instructions, never an independently stored fact set.""" return dict( - source_provenance=self.projected_source_provenance, - unit_interval_intensity=self.projected_unit_interval_intensity, + source_provenance=self.metadata.source_provenance, + unit_interval_intensity=self.metadata.unit_interval_intensity, ) @@ -1438,25 +1438,11 @@ class SourcePlaneImageMetadataProjection(ImageMetadataProjection): plane_index: int - @property - def projected_source_provenance(self) -> SourceImageProvenance: - return super( - SourcePlaneImageMetadataProjection, self - ).projected_source_provenance.for_source_plane(self.plane_index) - - @property - def projected_unit_interval_intensity( - self, - ) -> ImageUnitIntervalIntensityMetadata | None: - intensity = super( - SourcePlaneImageMetadataProjection, self - ).projected_unit_interval_intensity - return ( - None if intensity is None else intensity.for_source_plane(self.plane_index) - ) - def projected_fields(self) -> dict[str, Any]: changes = super(SourcePlaneImageMetadataProjection, self).projected_fields() + changes["source_provenance"] = changes["source_provenance"].for_source_plane( + self.plane_index + ) changes.update( intensity_scale=self.metadata.intensity_scale_for_source_plane( self.plane_index @@ -1465,45 +1451,48 @@ def projected_fields(self) -> dict[str, Any]: self.metadata.source_plane_dtypes, self.plane_index ) or self.metadata.source_dtype, + unit_interval_intensity=( + None + if changes["unit_interval_intensity"] is None + else changes["unit_interval_intensity"].for_source_plane( + self.plane_index + ) + ), source_plane_intensity_scales=(), source_plane_dtypes=(), ) + changes.update(self.metadata.normalized_fields(changes)) return changes class LeadingPlaneAxisMetadataProjection(ImageMetadataProjection): """Remove a declared leading axis while preserving its source contributors.""" - @property - def projected_source_provenance(self) -> SourceImageProvenance: - return super().projected_source_provenance.with_runtime_planes_as_contributors() - - @property - def projected_unit_interval_intensity( - self, - ) -> ImageUnitIntervalIntensityMetadata | None: - intensity = super().projected_unit_interval_intensity - return None if intensity is None else intensity.without_source_planes() - def projected_fields(self) -> dict[str, Any]: - self.metadata.require_leading_plane_axis( - "Image metadata has no leading plane axis to remove." - ) + if self.metadata.plane_axis is None: + raise ValueError("Image metadata has no leading plane axis to remove.") + changes = super().projected_fields() source_channel_axis = self.metadata.source_channel_axis if source_channel_axis == 0: raise ValueError( "Image metadata cannot declare the same leading axis as both " "plane and channel." ) - self.metadata.validate_source_channel_axis() if source_channel_axis is not None and source_channel_axis > 0: source_channel_axis -= 1 - changes = super().projected_fields() + changes["source_provenance"] = changes[ + "source_provenance" + ].with_runtime_planes_as_contributors() + changes.update(self.metadata.normalized_fields(changes)) + intensity = changes["unit_interval_intensity"] changes.update( plane_axis=None, source_channel_axis=source_channel_axis, source_plane_intensity_scales=(), source_plane_dtypes=(), + unit_interval_intensity=( + None if intensity is None else intensity.without_source_planes() + ), ) return changes @@ -1515,10 +1504,10 @@ class LeadingSourcePlaneMetadataProjection( """Compose source selection and axis removal before constructing metadata.""" def projected_fields(self) -> dict[str, Any]: - self.metadata.require_leading_plane_axis( - "Leading source-plane projection requires a declared plane axis." - ) - self.metadata.validate_source_channel_axis() + if self.metadata.plane_axis is None: + raise ValueError( + "Leading source-plane projection requires a declared plane axis." + ) return super().projected_fields() diff --git a/openhcs/core/source_image_provenance.py b/openhcs/core/source_image_provenance.py index 025d77b87f..366a8d5fd4 100644 --- a/openhcs/core/source_image_provenance.py +++ b/openhcs/core/source_image_provenance.py @@ -702,6 +702,15 @@ def __init__( self.source_image_names, ) + def normalized(self) -> "SourceImageProvenance": + """Snapshot provenance through the constructor's concrete field policy.""" + return SourceImageProvenance( + self.source_path, + self.source_component_metadata, + self.source_image_provenance_planes, + self.source_image_names, + ) + @classmethod def from_init_values( cls, @@ -1788,11 +1797,8 @@ def absorb_explicit_source_provenance( self.source_provenance = explicit.with_missing_from(self.source_provenance) def normalize_source_provenance_fields(self) -> None: - self.source_provenance = SourceImageProvenance( - self.source_provenance.source_path, - self.source_provenance.source_component_metadata, - self.source_provenance.source_image_provenance_planes, - self.source_provenance.source_image_names, + self.source_provenance = SourceImageProvenance.normalized( + self.source_provenance ) def replace_fields(self, **changes: Any) -> Self: diff --git a/openhcs/core/source_metadata.py b/openhcs/core/source_metadata.py index 2b819bb0e9..cdc93922fd 100644 --- a/openhcs/core/source_metadata.py +++ b/openhcs/core/source_metadata.py @@ -798,6 +798,13 @@ def __post_init__(self) -> None: if not isinstance(self.unit, SourceVoxelSpacingUnit): raise TypeError("SourceVoxelSpacing.unit must be SourceVoxelSpacingUnit.") + @classmethod + def coerce( + cls, value: "SourceVoxelSpacing | Sequence[float]" + ) -> "SourceVoxelSpacing": + """Retain nominal spacing or validate authored coordinate values.""" + return value if isinstance(value, cls) else cls(tuple(value)) + @property def has_values(self) -> bool: return bool(self.values_zyx) @@ -960,10 +967,7 @@ class SourceVoxelSpacingFields: ) def normalize_source_voxel_spacing_fields(self) -> None: - if not isinstance(self.source_voxel_spacing, SourceVoxelSpacing): - self.source_voxel_spacing = SourceVoxelSpacing( - tuple(self.source_voxel_spacing) - ) + self.source_voxel_spacing = SourceVoxelSpacing.coerce(self.source_voxel_spacing) @lru_cache(maxsize=4096) diff --git a/tests/unit/test_image_payload_projection_algebra.py b/tests/unit/test_image_payload_projection_algebra.py index ef898d9870..23bbc68f1c 100644 --- a/tests/unit/test_image_payload_projection_algebra.py +++ b/tests/unit/test_image_payload_projection_algebra.py @@ -19,6 +19,7 @@ SourceImageProvenanceContributor, SourceImageProvenancePlanes, ) +from openhcs.core.source_spatial_domain import SourceSpatialDomain def _metadata(axis, channel_axis): @@ -152,3 +153,165 @@ def test_projection_observes_mask_and_metadata_mutation(): mask.resize((2, 2, 6), refcheck=False) with pytest.raises(ValueError, match="cannot be projected"): projector.payload_for_slice(data, 1) + + +def _metadata_with_observed_plane_fields(*, proof_raises=False): + events = [] + + class ObservedInt(int): + def __int__(self): + events.append("proof-int") + if proof_raises: + raise RuntimeError("proof conversion observed") + return 1 + + class ObservedTuple(tuple): + def __getitem__(self, index): + events.append("intensity-getitem") + return super().__getitem__(index) + + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_plane_intensity_scales=ObservedTuple((2.0,)), + unit_interval_intensity=ImageUnitIntervalIntensityMetadata( + source_plane_scales=(ObservedInt(1),) + ), + ) + return metadata, events + + +def test_plane_proof_failure_precedes_leading_channel_guard(): + metadata, events = _metadata_with_observed_plane_fields(proof_raises=True) + metadata.source_channel_axis = 0 + with pytest.raises(RuntimeError, match="proof conversion observed"): + metadata.for_leading_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + + +def test_plane_field_effects_precede_leading_channel_guard(): + metadata, events = _metadata_with_observed_plane_fields() + metadata.source_channel_axis = 0 + with pytest.raises(ValueError, match="both plane and channel"): + metadata.for_leading_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + + +def test_source_plane_preserves_intensity_then_proof_effect_order(): + metadata, events = _metadata_with_observed_plane_fields() + projected = metadata.for_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + assert projected.intensity_scale == 2.0 + assert projected.unit_interval_intensity.scale == 1 + + +def test_plane_proof_failure_precedes_mutated_channel_type_guard(): + metadata, events = _metadata_with_observed_plane_fields(proof_raises=True) + metadata.source_channel_axis = False + with pytest.raises(RuntimeError, match="proof conversion observed"): + metadata.for_leading_source_plane(0) + assert events == ["intensity-getitem", "proof-int"] + + +@pytest.mark.parametrize( + "operation,channel_axis,broken,error,message", + ( + ("for_leading_source_plane", 0, "domain", ValueError, "spatial_origin_yx"), + ("for_leading_source_plane", 0, "spacing", ValueError, "finite and positive"), + ( + "for_leading_source_plane", + 0, + "spacing-value", + AttributeError, + "with_missing_from", + ), + ("for_leading_source_plane", 0, "plane", ValueError, "invalid-axis"), + ("for_leading_source_plane", False, "domain", ValueError, "spatial_origin_yx"), + ( + "for_leading_source_plane", + False, + "spacing", + ValueError, + "finite and positive", + ), + ("for_leading_source_plane", False, "plane", TypeError, "must be int or None"), + ("for_source_plane", False, "domain", ValueError, "spatial_origin_yx"), + ("for_source_plane", False, "spacing", ValueError, "finite and positive"), + ( + "for_source_plane", + False, + "spacing-value", + AttributeError, + "with_missing_from", + ), + ("for_source_plane", False, "plane", TypeError, "must be int or None"), + ( + "without_leading_plane_axis", + 0, + "domain", + ValueError, + "both plane and channel", + ), + ( + "without_leading_plane_axis", + 0, + "spacing", + ValueError, + "both plane and channel", + ), + ( + "without_leading_plane_axis", + 0, + "spacing-value", + ValueError, + "both plane and channel", + ), + ( + "without_leading_plane_axis", + 0, + "plane", + ValueError, + "both plane and channel", + ), + ("without_leading_plane_axis", "1", "plane", TypeError, "not supported"), + ), +) +def test_projection_preserves_cross_invalid_constructor_guard_order( + operation, channel_axis, broken, error, message +): + metadata = ImagePayloadMetadata( + source_path="/tmp/source.tif", + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + ) + metadata.source_channel_axis = channel_axis + if broken == "domain": + metadata.source_spatial_domain = SourceSpatialDomain(origin_yx=(1,)) + elif broken == "spacing": + metadata.source_component_metadata = {"OpenHCSSourceVoxelSpacingZYX": "0,1,1"} + elif broken == "spacing-value": + metadata.source_voxel_spacing = None + else: + metadata.plane_axis = "invalid-axis" + with pytest.raises(error, match=message): + if operation == "without_leading_plane_axis": + metadata.without_leading_plane_axis() + else: + getattr(metadata, operation)(0) + + +def test_missing_leading_axis_precedes_plane_field_effects(): + metadata, events = _metadata_with_observed_plane_fields(proof_raises=True) + metadata.plane_axis = None + metadata.source_channel_axis = 0 + with pytest.raises(ValueError, match="requires a declared plane axis"): + metadata.for_leading_source_plane(0) + assert events == [] + + +def test_projection_spacing_failure_precedes_other_invalid_source_fields(): + metadata = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE) + metadata.source_component_metadata = {"OpenHCSSourceVoxelSpacingZYX": "0,1,1"} + metadata.source_spatial_domain = SourceSpatialDomain(origin_yx=(1,)) + metadata.source_channel_axis = False + metadata.plane_axis = "invalid-axis" + with pytest.raises(ValueError, match="finite and positive"): + metadata.for_leading_source_plane(0) From 5bc21242352c08f17e6e9815d2a6dbb000141b3d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 21:43:31 -0400 Subject: [PATCH 030/125] Record complete-owner execution gain and cold compilation regression for #384 --- .../shared_runtime_plumbing_384_20261001.rst | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index eae4701dee..b897c1393e 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -396,3 +396,60 @@ hooks. The next gate is an ordinary production comparison against the existing nominal immutable-field owner, before further implementation or broad gates. Evidence: ``metadata-lifetime-union-production-counterfactual-admission-20261001.json`` and ``metadata-lifetime-union-reflection-pair-control-20261001.json``. + +Initial ordinary complete-field-owner counterfactual +--------------------------------------------------- + +The existing immutable field-owner migration is now included in this draft, +rather than left on an unpublished independent branch. ``SourceMetadataFields`` +owns the common algorithms; live declared fields and owned resolved/durable +fields provide their actual lifetime through inheritance. Owned builtin values +permit local derived-view reuse, indexed lookup and readonly snapshot reuse. +Raw mutable mappings retain fresh derivation. The obsolete role/identity view +types and duplicate field algorithms are removed. Fields-only transport drops +process-local derived caches. + +One initial ordinary, uninstrumented 3D pair compares main +9a04107492ad90233394cf17524d0ca8e74062bb against the complete candidate +401cd69fa718d6b19ce4dc35fb5f7c2d70f2d219: + +=================== ============ ============ +Metric Main Candidate +=================== ============ ============ +Execution 11.6153 s 8.1170 s +Compilation 2.2212 s 50.7997 s +Pipeline total 14.5439 s 59.6670 s +=================== ============ ============ + +Execution improves by 3.4984 seconds in this initial pair; total time regresses +by 45.1231 seconds. All six actual result CSVs have exact rows and all 120 TIFF +arrays have exact dtype, shape and pixels, with equal complete inventories. +This is an initial counterfactual, not accepted ABBA or all-case native parity. + +The compilation regression is not startup time: 239 Numba cache files are written +after axis compilation, over 48.2219 seconds, ending immediately before compilation +completes. The cold candidate source-path kernel cache exposes a startup regression. +Commit 2b86bfbe49f36d302dcd67c5b0a0cb4f656f1eb0 implemented registry kernel warming; +ffba8426cefcaa87b6b00ff056e4cb96855233e4 in PR358 subsequently removed that loop +and deferred readiness to selected compilation. Restoring the existing registry +warmup lifecycle is tracked by issue162. No total-time improvement is claimed +until that repair is verified with fresh caches. + +The field-owner integration into this draft passes 81 scoped lifetime/ownership +controls with metaclass-registry 0.2.2. A subsequent projection correctness repair +passes 173 scoped controls in its isolated source. It restores source-plane +derivation and cross-invalid field/error ordering before leading-axis guards, +shares normalization on the existing metadata owner, and deletes premature +projection properties. That repair requires performance requalification; the +initial pair is not projected through it. Arbitrary external constructor/mapping +callback equivalence remains an OPEN obligation, not a claimed proof. + +Evidence: ``metadata-owner-ordinary-pair-output-parity-20261001.json``, +``metadata-owner-cold-compilation-regression-cause-20261001.json``, +``source-projection-order-384-correctness-patch-20261002.json``, +``/var/tmp/openhcs-metadata-owner-ordinary-pair-20261001/observations.json`` and +``/var/tmp/openhcs-integrated-metadata-owner-scoped-gates-20261002.log``. +Main 25d56ae3fb9b80acda80f3cf4e1c8667939147eb is normally merged; the shared +environment and PR submodule consume the declared published metaclass-registry +0.2.2 source 393a7e03003cdc56df9013f932ed4f26e632d77a. Earlier observations retain +their measured source/dependency pins. From 02609e12b41669b63c11526496eecadbb7fbc87c Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 22:01:13 -0400 Subject: [PATCH 031/125] Record verified empty-cache startup readiness restoration Retain the cold compilation failure and distinguish the main lifecycle fix from integrated candidate performance. Refs #384 and #162. --- .../shared_runtime_plumbing_384_20261001.rst | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index b897c1393e..4ed582c9e5 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -431,9 +431,9 @@ after axis compilation, over 48.2219 seconds, ending immediately before compilat completes. The cold candidate source-path kernel cache exposes a startup regression. Commit 2b86bfbe49f36d302dcd67c5b0a0cb4f656f1eb0 implemented registry kernel warming; ffba8426cefcaa87b6b00ff056e4cb96855233e4 in PR358 subsequently removed that loop -and deferred readiness to selected compilation. Restoring the existing registry -warmup lifecycle is tracked by issue162. No total-time improvement is claimed -until that repair is verified with fresh caches. +and deferred readiness to selected compilation. PR420 restores the existing registry warmup lifecycle and closes issue162. +Its separate empty-cache regression gate is recorded below; the integrated +field-owner performance still requires requalification. The field-owner integration into this draft passes 81 scoped lifetime/ownership controls with metaclass-registry 0.2.2. A subsequent projection correctness repair @@ -453,3 +453,35 @@ Main 25d56ae3fb9b80acda80f3cf4e1c8667939147eb is normally merged; the shared environment and PR submodule consume the declared published metaclass-registry 0.2.2 source 393a7e03003cdc56df9013f932ed4f26e632d77a. Earlier observations retain their measured source/dependency pins. + +Restored registry readiness on main +---------------------------------- + +PR420 merges the startup correction to main +107498cfacf3b12d926c55f84c4b3cb7c04aa50f. That main is normally merged into +this performance branch. Existing affinity admission, cancellation ownership +and compiler guards for declarations introduced after startup are retained. + +An ordinary, uninstrumented 3D 1w_1t run at fix source +e55e2ec0bc98ad750837cd4dcfd830ad04096286 starts with a new empty Numba cache. +Compilation is 1.720214 s, execution 11.151797 s and pipeline total 13.637182 s. +All 481 cache files are written before pipeline submission; the last write is +6.001317 s before run creation and no cache files are written during compilation +or execution. Server startup, warmup and shutdown are excluded from pipeline +clocks. All six CSVs and 120 TIFF arrays exactly match retained main9a output, +including dtype, shape and matching inventories. This is a lifecycle regression +gate, not an ABBA performance comparison or current integrated-candidate timing. + +The fix passes 37 scoped tests with five single-affinity skips and both actual +two-core fork controls. Original unmodified R0 passes for openhcs, scripts and +benchmark; R1 reports no increases against main25d56ae under its unchanged +160-second budget. The failed Python3.12 ratchet launch and uninitialized +worktree R1 launch remain retained; final guards use Python3.14 and the initialized +main repository, respectively, without modifying either detector. + +Evidence: ``registry-startup-readiness-162-empty-cache-pipeline-20261002.json``, +``registry-startup-readiness-162-validation-20261001.json`` and +``registry-startup-readiness-162-original-r0-py314-20261002.log`` plus +``registry-startup-readiness-162-original-r1-initialized-root-20261002.log``. +The original 50.800-second compilation regression remains retained. Current +combined performance, IFC behavior and source guards remain draft gates. From 5337fda87b571580862d917fde95a5397cb5e4fe Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 22:27:12 -0400 Subject: [PATCH 032/125] Share scalar admission through nominal source field ownership Canonical and durable owners retain their normalization and error policies while sharing one primitive admission algorithm. Preserve observed scalar class checks and remove duplicate validation. Refs #384. --- openhcs/core/source_metadata.py | 47 ++++++++++++++++-------- tests/unit/test_source_metadata_owner.py | 19 ++++++++++ 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/openhcs/core/source_metadata.py b/openhcs/core/source_metadata.py index cdc93922fd..c8ab6df14b 100644 --- a/openhcs/core/source_metadata.py +++ b/openhcs/core/source_metadata.py @@ -10,7 +10,7 @@ from math import isfinite from pathlib import Path from types import MappingProxyType -from typing import ClassVar, Self, TYPE_CHECKING, TypeAlias, TypeVar +from typing import ClassVar, NoReturn, Self, TYPE_CHECKING, TypeAlias, TypeVar from metaclass_registry import AutoRegisterMeta from zmqruntime.viewer_protocol import ViewerWireField @@ -77,9 +77,29 @@ def normalized_mapping( {str(key): cls.normalized_value(value) for key, value in metadata.items()} ) + @classmethod + def normalized_scalar(cls, value: SourceMetadataScalar) -> SourceMetadataScalar: + """Admit the shared scalar grammar before applying the owner's policy.""" + if value is None: + return None + if isinstance(value, SourceMetadataNonNullScalar): + return cls._normalize_admitted_scalar(value) + return cls._reject_scalar(value) + + @staticmethod + def _normalize_admitted_scalar( + value: SourceMetadataNonNullScalar, + ) -> SourceMetadataNonNullScalar: + if isinstance(value, str): + return canonical_path_metadata_value(value) + return value + @staticmethod - def normalized_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: - return source_metadata_scalar(value) + def _reject_scalar(value: object) -> NoReturn: + raise TypeError( + "Source metadata scalar values must be str, int, float, bool, or None, " + f"got {type(value).__name__}." + ) def __getitem__(self, key: str) -> SourceMetadataValue: for field_key, value in self.fields: @@ -499,9 +519,13 @@ def __eq__(self, other: object) -> bool: __hash__ = None @staticmethod - def normalized_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: - if value is None or isinstance(value, SourceMetadataNonNullScalar): - return value + def _normalize_admitted_scalar( + value: SourceMetadataNonNullScalar, + ) -> SourceMetadataNonNullScalar: + return value + + @staticmethod + def _reject_scalar(value: object) -> NoReturn: if isinstance(value, Mapping) or ( isinstance(value, Sequence) and not isinstance(value, str) ): @@ -546,16 +570,7 @@ def source_metadata_dict( def source_metadata_scalar(value: SourceMetadataScalar) -> SourceMetadataScalar: """Return the canonical scalar representation stored in source metadata.""" - if value is None: - return None - if not isinstance(value, SourceMetadataNonNullScalar): - raise TypeError( - "Source metadata scalar values must be str, int, float, bool, or None, " - f"got {type(value).__name__}." - ) - if isinstance(value, str): - return canonical_path_metadata_value(value) - return value + return SourceMetadataFields.normalized_scalar(value) def canonical_path_metadata_value(value: str) -> str: diff --git a/tests/unit/test_source_metadata_owner.py b/tests/unit/test_source_metadata_owner.py index a092129dfd..4a492b864d 100644 --- a/tests/unit/test_source_metadata_owner.py +++ b/tests/unit/test_source_metadata_owner.py @@ -587,6 +587,25 @@ def __class__(self): assert DurableSourceMetadata.normalized_scalar(reported_integer) is reported_integer assert reported_integer.reads == [object, int] + for normalize in ( + source_metadata_scalar, + SourceMetadataFields.normalized_scalar, + ResolvedSourceMetadataRecord.normalized_scalar, + ): + reported_integer = ReportedClass((object, int, object)) + assert normalize(reported_integer) is reported_integer + assert reported_integer.reads == [object, int, object] + + # Scalar admission and string normalization are separate observations. + # A changing reported class must retain the previous normalization order. + reported_string = ReportedClass((str, object)) + assert normalize(reported_string) is reported_string + assert reported_string.reads == [str, object] + + reported_string = ReportedClass((str, object)) + assert DurableSourceMetadata.normalized_scalar(reported_string) is reported_string + assert reported_string.reads == [str] + changing_class = ReportedClass( (object, object, object, object, object, Sequence, str) ) From 80d92db0edb472dd8c81a169bc58d8549f4fb95c Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 22:35:32 -0400 Subject: [PATCH 033/125] Keep projection normalization on owned image metadata Compose source selection and axis transformation on one independently owned metadata result. Retain constructor effect ordering through typed field owners, remove redundant dictionary state and provenance forwarding, and preserve input isolation. Refs #384. --- openhcs/core/runtime_image_values.py | 176 ++++++++++-------- openhcs/core/source_image_provenance.py | 16 +- .../test_image_payload_projection_algebra.py | 111 +++++++++++ 3 files changed, 212 insertions(+), 91 deletions(-) diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index 5963e79799..896de868c3 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -239,33 +239,24 @@ def for_array_payload( def __post_init__(self, *source_provenance_values: object) -> None: self.absorb_explicit_source_provenance(source_provenance_values) - for name, value in self.normalized_fields({}).items(): - setattr(self, name, value) - - def normalized_fields(self, changes: Mapping[str, Any]) -> dict[str, Any]: - """Normalize constructor instructions in their source-context effect order.""" - provenance = changes.get("source_provenance", self.source_provenance) - spacing = changes.get("source_voxel_spacing", self.source_voxel_spacing) - spacing = spacing.with_missing_from( - SourceVoxelSpacing.from_source_metadata( - provenance.source_component_metadata - ) + self.normalize_metadata_fields() + + def normalize_metadata_fields(self) -> None: + """Normalize this metadata's typed fields in their constructor effect order.""" + self.source_voxel_spacing = self.source_voxel_spacing.with_missing_from( + SourceVoxelSpacing.from_source_metadata(self.source_component_metadata) ) - provenance = SourceImageProvenance.normalized(provenance) - domain = changes.get( - "source_spatial_domain", self.source_spatial_domain - ).normalized() - spacing = SourceVoxelSpacing.coerce(spacing) + self.normalize_source_provenance_fields() + self.normalize_source_spatial_domain_fields() + self.normalize_source_voxel_spacing_fields() self.validate_source_channel_axis() - plane_axis = changes.get("plane_axis", self.plane_axis) - if plane_axis is not None: - plane_axis = RuntimePlaneAxis(plane_axis) - return dict( - source_provenance=provenance, - source_voxel_spacing=spacing, - source_spatial_domain=domain, - plane_axis=plane_axis, - ) + if self.plane_axis is not None: + self.plane_axis = RuntimePlaneAxis(self.plane_axis) + + def require_leading_plane_axis(self, message: str) -> None: + """Require axis presence before later ordered projection validation.""" + if self.plane_axis is None: + raise ValueError(message) def validate_source_channel_axis(self) -> None: """Validate the authored channel declaration before transforming axes.""" @@ -657,9 +648,37 @@ def for_leading_source_plane(self, plane_index: int) -> "ImagePayloadMetadata": """Project metadata after explicitly removing its leading plane axis.""" return LeadingSourcePlaneMetadataProjection(self, plane_index).project() - def without_leading_plane_axis(self) -> "ImagePayloadMetadata": - """Return metadata after an explicitly declared leading axis is removed.""" - return LeadingPlaneAxisMetadataProjection(self).project() + def without_leading_plane_axis( + self, *, projection: "ImageMetadataProjection | None" = None + ) -> "ImagePayloadMetadata": + """Remove an axis through the projection's declared result ownership.""" + self.require_leading_plane_axis( + "Image metadata has no leading plane axis to remove." + ) + source_channel_axis = self.source_channel_axis + if source_channel_axis == 0: + raise ValueError( + "Image metadata cannot declare the same leading axis as both " + "plane and channel." + ) + if source_channel_axis is not None and source_channel_axis > 0: + source_channel_axis -= 1 + if projection is None: + projection = LeadingPlaneAxisMetadataProjection(self) + projected = projection.project_source_provenance( + self, self.source_provenance.with_runtime_planes_as_contributors() + ) + projected.plane_axis = None + projected.source_channel_axis = source_channel_axis + projected.source_plane_intensity_scales = () + projected.source_plane_dtypes = () + projected.unit_interval_intensity = ( + None + if self.unit_interval_intensity is None + else self.unit_interval_intensity.without_source_planes() + ) + projected.normalize_metadata_fields() + return projected def collapse_leading_plane_axis(self) -> "ImagePayloadMetadata": """Return scalar metadata after reducing every plane of the leading axis.""" @@ -1421,15 +1440,23 @@ class ImageMetadataProjection(ABC): metadata: ImagePayloadMetadata def project(self) -> ImagePayloadMetadata: - return self.metadata.replace_fields(**self.projected_fields()) + return self.project_axis_metadata(self.project_source_metadata()) @abstractmethod - def projected_fields(self) -> dict[str, Any]: - """Derive constructor instructions, never an independently stored fact set.""" - return dict( - source_provenance=self.metadata.source_provenance, - unit_interval_intensity=self.metadata.unit_interval_intensity, - ) + def project_source_metadata(self) -> ImagePayloadMetadata: + """Select source metadata with this projection's declared ownership.""" + + def project_axis_metadata( + self, metadata: ImagePayloadMetadata + ) -> ImagePayloadMetadata: + """Retain declared axes unless a nominal axis capability transforms them.""" + return metadata + + @abstractmethod + def project_source_provenance( + self, metadata: ImagePayloadMetadata, provenance: SourceImageProvenance + ) -> ImagePayloadMetadata: + """Apply provenance using the result ownership established by source selection.""" @dataclass(frozen=True, slots=True) @@ -1438,12 +1465,9 @@ class SourcePlaneImageMetadataProjection(ImageMetadataProjection): plane_index: int - def projected_fields(self) -> dict[str, Any]: - changes = super(SourcePlaneImageMetadataProjection, self).projected_fields() - changes["source_provenance"] = changes["source_provenance"].for_source_plane( - self.plane_index - ) - changes.update( + def project_source_metadata(self) -> ImagePayloadMetadata: + provenance = self.metadata.source_provenance.for_source_plane(self.plane_index) + return self.metadata.replace_fields( intensity_scale=self.metadata.intensity_scale_for_source_plane( self.plane_index ), @@ -1451,64 +1475,56 @@ def projected_fields(self) -> dict[str, Any]: self.metadata.source_plane_dtypes, self.plane_index ) or self.metadata.source_dtype, + source_provenance=provenance, unit_interval_intensity=( None - if changes["unit_interval_intensity"] is None - else changes["unit_interval_intensity"].for_source_plane( + if self.metadata.unit_interval_intensity is None + else self.metadata.unit_interval_intensity.for_source_plane( self.plane_index ) ), source_plane_intensity_scales=(), source_plane_dtypes=(), ) - changes.update(self.metadata.normalized_fields(changes)) - return changes + + def project_source_provenance( + self, metadata: ImagePayloadMetadata, provenance: SourceImageProvenance + ) -> ImagePayloadMetadata: + """Normalize the independently owned result created by source selection.""" + metadata.source_provenance = provenance + metadata.normalize_metadata_fields() + return metadata class LeadingPlaneAxisMetadataProjection(ImageMetadataProjection): """Remove a declared leading axis while preserving its source contributors.""" - def projected_fields(self) -> dict[str, Any]: - if self.metadata.plane_axis is None: - raise ValueError("Image metadata has no leading plane axis to remove.") - changes = super().projected_fields() - source_channel_axis = self.metadata.source_channel_axis - if source_channel_axis == 0: - raise ValueError( - "Image metadata cannot declare the same leading axis as both " - "plane and channel." - ) - if source_channel_axis is not None and source_channel_axis > 0: - source_channel_axis -= 1 - changes["source_provenance"] = changes[ - "source_provenance" - ].with_runtime_planes_as_contributors() - changes.update(self.metadata.normalized_fields(changes)) - intensity = changes["unit_interval_intensity"] - changes.update( - plane_axis=None, - source_channel_axis=source_channel_axis, - source_plane_intensity_scales=(), - source_plane_dtypes=(), - unit_interval_intensity=( - None if intensity is None else intensity.without_source_planes() - ), - ) - return changes + def project_source_metadata(self) -> ImagePayloadMetadata: + return self.metadata + + def project_axis_metadata( + self, metadata: ImagePayloadMetadata + ) -> ImagePayloadMetadata: + return metadata.without_leading_plane_axis(projection=self) + + def project_source_provenance( + self, metadata: ImagePayloadMetadata, provenance: SourceImageProvenance + ) -> ImagePayloadMetadata: + """Create the standalone result after its axis guards and source derivation.""" + return metadata.replace_fields(source_provenance=provenance) class LeadingSourcePlaneMetadataProjection( - LeadingPlaneAxisMetadataProjection, SourcePlaneImageMetadataProjection, + LeadingPlaneAxisMetadataProjection, ): - """Compose source selection and axis removal before constructing metadata.""" + """Remove the leading axis of independently owned selected-source metadata.""" - def projected_fields(self) -> dict[str, Any]: - if self.metadata.plane_axis is None: - raise ValueError( - "Leading source-plane projection requires a declared plane axis." - ) - return super().projected_fields() + def project_source_metadata(self) -> ImagePayloadMetadata: + self.metadata.require_leading_plane_axis( + "Leading source-plane projection requires a declared plane axis." + ) + return super().project_source_metadata() @dataclass(frozen=True, slots=True) diff --git a/openhcs/core/source_image_provenance.py b/openhcs/core/source_image_provenance.py index 366a8d5fd4..025d77b87f 100644 --- a/openhcs/core/source_image_provenance.py +++ b/openhcs/core/source_image_provenance.py @@ -702,15 +702,6 @@ def __init__( self.source_image_names, ) - def normalized(self) -> "SourceImageProvenance": - """Snapshot provenance through the constructor's concrete field policy.""" - return SourceImageProvenance( - self.source_path, - self.source_component_metadata, - self.source_image_provenance_planes, - self.source_image_names, - ) - @classmethod def from_init_values( cls, @@ -1797,8 +1788,11 @@ def absorb_explicit_source_provenance( self.source_provenance = explicit.with_missing_from(self.source_provenance) def normalize_source_provenance_fields(self) -> None: - self.source_provenance = SourceImageProvenance.normalized( - self.source_provenance + self.source_provenance = SourceImageProvenance( + self.source_provenance.source_path, + self.source_provenance.source_component_metadata, + self.source_provenance.source_image_provenance_planes, + self.source_provenance.source_image_names, ) def replace_fields(self, **changes: Any) -> Self: diff --git a/tests/unit/test_image_payload_projection_algebra.py b/tests/unit/test_image_payload_projection_algebra.py index 23bbc68f1c..086520fd4c 100644 --- a/tests/unit/test_image_payload_projection_algebra.py +++ b/tests/unit/test_image_payload_projection_algebra.py @@ -315,3 +315,114 @@ def test_projection_spacing_failure_precedes_other_invalid_source_fields(): metadata.plane_axis = "invalid-axis" with pytest.raises(ValueError, match="finite and positive"): metadata.for_leading_source_plane(0) + + +@pytest.mark.parametrize( + "operation,normalization_axes", + ( + ("without_leading_plane_axis", (RuntimePlaneAxis.RUNTIME_SLICE, None)), + ( + "for_leading_source_plane", + (RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.RUNTIME_SLICE, None), + ), + ), +) +def test_projection_normalizes_owned_metadata_at_each_historical_phase( + monkeypatch, operation, normalization_axes +): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, 2) + input_provenance = metadata.source_provenance + input_proof = metadata.unit_interval_intensity + births = [] + phases = [] + original_constructor = ImagePayloadMetadata.__post_init__ + original_normalize = ImagePayloadMetadata.normalize_metadata_fields + + def observe_constructor(self, *values): + births.append((self, self.plane_axis, self.source_channel_axis)) + original_constructor(self, *values) + + def observe_normalization(self): + phases.append((self, self.plane_axis, self.source_channel_axis)) + original_normalize(self) + + monkeypatch.setattr(ImagePayloadMetadata, "__post_init__", observe_constructor) + monkeypatch.setattr( + ImagePayloadMetadata, "normalize_metadata_fields", observe_normalization + ) + projected = ( + metadata.without_leading_plane_axis() + if operation == "without_leading_plane_axis" + else metadata.for_leading_source_plane(1) + ) + + assert births == [(projected, RuntimePlaneAxis.RUNTIME_SLICE, 2)] + assert tuple(axis for _, axis, _ in phases) == normalization_axes + assert all(owner is projected for owner, _, _ in phases) + assert tuple(channel for _, _, channel in phases) == (2,) * ( + len(normalization_axes) - 1 + ) + (1,) + assert projected is not metadata + assert metadata.source_provenance is input_provenance + assert metadata.unit_interval_intensity is input_proof + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_channel_axis == 2 + assert metadata.source_plane_intensity_scales == (255.0, 65535.0) + assert projected.source_provenance is not input_provenance + assert projected.unit_interval_intensity is not input_proof + + +@pytest.mark.parametrize("combined", (False, True)) +def test_projection_preserves_contributor_derivation_constructor_error_order( + monkeypatch, combined +): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, None) + invalid_domain = SourceSpatialDomain(origin_yx=(1,)) + metadata.source_spatial_domain = invalid_domain + provenance = metadata.source_provenance + calls = [] + + def fail_contributors(self): + calls.append(self) + raise RuntimeError("contributor derivation observed") + + monkeypatch.setattr( + type(provenance), "with_runtime_planes_as_contributors", fail_contributors + ) + if combined: + with pytest.raises(ValueError, match="spatial_origin_yx"): + metadata.for_leading_source_plane(1) + assert calls == [] + else: + with pytest.raises(RuntimeError, match="contributor derivation observed"): + metadata.without_leading_plane_axis() + assert calls == [provenance] + assert metadata.source_provenance is provenance + assert metadata.source_spatial_domain is invalid_domain + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_plane_intensity_scales == (255.0, 65535.0) + + +@pytest.mark.parametrize("combined", (False, True)) +def test_failed_owned_axis_transform_does_not_mutate_source(monkeypatch, combined): + metadata = _metadata(RuntimePlaneAxis.RUNTIME_SLICE, 2) + provenance = metadata.source_provenance + proof = metadata.unit_interval_intensity + + def fail_proof(self): + raise RuntimeError("axis proof derivation observed") + + monkeypatch.setattr( + ImageUnitIntervalIntensityMetadata, "without_source_planes", fail_proof + ) + with pytest.raises(RuntimeError, match="axis proof derivation observed"): + if combined: + metadata.for_leading_source_plane(1) + else: + metadata.without_leading_plane_axis() + assert metadata.source_provenance is provenance + assert metadata.unit_interval_intensity is proof + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_channel_axis == 2 + assert metadata.source_plane_intensity_scales == (255.0, 65535.0) + assert metadata.source_plane_dtypes == ("uint8", "uint16") From 90c30e1adfacd25a834ed48949b430099c866134 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 22:51:53 -0400 Subject: [PATCH 034/125] Share nullable intensity proof projection on image metadata Address #384 by consolidating source-plane and axis-removal proof handling on the existing metadata owner, and deriving scale queries from their existing scalar and quantization authorities. --- openhcs/core/runtime_image_values.py | 39 +++++------ .../test_image_payload_projection_algebra.py | 67 +++++++++++++++++++ 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/openhcs/core/runtime_image_values.py b/openhcs/core/runtime_image_values.py index 896de868c3..6ac2a80ae6 100644 --- a/openhcs/core/runtime_image_values.py +++ b/openhcs/core/runtime_image_values.py @@ -672,11 +672,7 @@ def without_leading_plane_axis( projected.source_channel_axis = source_channel_axis projected.source_plane_intensity_scales = () projected.source_plane_dtypes = () - projected.unit_interval_intensity = ( - None - if self.unit_interval_intensity is None - else self.unit_interval_intensity.without_source_planes() - ) + projected.unit_interval_intensity = self.project_intensity_proof(None) projected.normalize_metadata_fields() return projected @@ -714,22 +710,27 @@ def source_image_paths(self) -> tuple[str, ...]: def intensity_scale_for_source_plane(self, plane_index: int) -> float | None: """Return the best available intensity scale for one source plane.""" - if 0 <= plane_index < len(self.source_plane_intensity_scales): - plane_scale = self.source_plane_intensity_scales[plane_index] - if plane_scale is not None: - return plane_scale - return self.intensity_scale + plane_scale = _tuple_value(self.source_plane_intensity_scales, plane_index) + return self.intensity_scale if plane_scale is None else plane_scale def unit_interval_intensity_scale_for_source_plane( self, plane_index: int, ) -> int | None: """Return the scale proving current pixels are exact integer/scale values.""" - if 0 <= plane_index < len(self.source_plane_unit_interval_intensity_scales): - plane_scale = self.source_plane_unit_interval_intensity_scales[plane_index] - if plane_scale is not None: - return int(plane_scale) - return self.unit_interval_intensity_scale + if self.unit_interval_intensity is None: + return None + return self.unit_interval_intensity.scale_for_source_plane(plane_index) + + def project_intensity_proof( + self, plane_index: int | None + ) -> ImageUnitIntervalIntensityMetadata | None: + """Select a plane's proof, or retain only the proof for the whole image.""" + if self.unit_interval_intensity is None: + return None + if plane_index is None: + return self.unit_interval_intensity.without_source_planes() + return self.unit_interval_intensity.for_source_plane(plane_index) @property def source_plane_metadata_count(self) -> int: @@ -1476,12 +1477,8 @@ def project_source_metadata(self) -> ImagePayloadMetadata: ) or self.metadata.source_dtype, source_provenance=provenance, - unit_interval_intensity=( - None - if self.metadata.unit_interval_intensity is None - else self.metadata.unit_interval_intensity.for_source_plane( - self.plane_index - ) + unit_interval_intensity=self.metadata.project_intensity_proof( + self.plane_index ), source_plane_intensity_scales=(), source_plane_dtypes=(), diff --git a/tests/unit/test_image_payload_projection_algebra.py b/tests/unit/test_image_payload_projection_algebra.py index 086520fd4c..677ed7beb1 100644 --- a/tests/unit/test_image_payload_projection_algebra.py +++ b/tests/unit/test_image_payload_projection_algebra.py @@ -426,3 +426,70 @@ def fail_proof(self): assert metadata.source_channel_axis == 2 assert metadata.source_plane_intensity_scales == (255.0, 65535.0) assert metadata.source_plane_dtypes == ("uint8", "uint16") + + +@pytest.mark.parametrize( + "plane_index,expected_scale", + ((None, 255), (-1, 255), (0, 255), (1, 65535), (2, 255)), +) +def test_owned_proof_projection_preserves_scalar_fallback_and_axis_removal( + plane_index, expected_scale +): + proof = ImageUnitIntervalIntensityMetadata( + scale=255, source_plane_scales=(None, 65535) + ) + metadata = ImagePayloadMetadata(unit_interval_intensity=proof) + projected = metadata.project_intensity_proof(plane_index) + assert projected is not proof + assert projected.scale == expected_scale + assert projected.source_plane_scales == () + assert metadata.unit_interval_intensity is proof + assert proof.source_plane_scales == (None, 65535) + if plane_index is not None: + assert metadata.unit_interval_intensity_scale_for_source_plane(plane_index) == ( + expected_scale + ) + + +@pytest.mark.parametrize("plane_index", (None, 0, 1)) +def test_owned_proof_projection_observes_absence_and_live_replacement(plane_index): + metadata = ImagePayloadMetadata(intensity_scale=255) + assert metadata.project_intensity_proof(plane_index) is None + if plane_index is not None: + assert ( + metadata.unit_interval_intensity_scale_for_source_plane(plane_index) is None + ) + proof = ImageUnitIntervalIntensityMetadata(scale=7, source_plane_scales=(11, 13)) + metadata.unit_interval_intensity = proof + assert metadata.project_intensity_proof(plane_index).scale == ( + 7 if plane_index is None else (11, 13)[plane_index] + ) + metadata.unit_interval_intensity = None + assert metadata.project_intensity_proof(plane_index) is None + + +def test_quantization_queries_preserve_integer_conversion_and_fallback_identity(): + calls = [] + + class ObservedInt(int): + def __int__(self): + calls.append(self) + return super().__int__() + + scalar = ObservedInt(255) + plane = ObservedInt(65535) + metadata = ImagePayloadMetadata( + unit_interval_intensity=ImageUnitIntervalIntensityMetadata( + scale=scalar, source_plane_scales=(None, plane) + ) + ) + assert metadata.unit_interval_intensity_scale_for_source_plane(0) is scalar + assert metadata.unit_interval_intensity_scale_for_source_plane(-1) is scalar + assert calls == [] + assert metadata.unit_interval_intensity_scale_for_source_plane(1) == 65535 + assert calls == [plane] + calls.clear() + assert metadata.project_intensity_proof(1).scale == 65535 + assert calls == [plane] + assert metadata.project_intensity_proof(None).scale is scalar + assert calls == [plane] From 37726ea29c7d2e5efed210bf51544d167fe72d66 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 23:00:25 -0400 Subject: [PATCH 035/125] Check slice projection on concrete runtime parameter declarations Preserve strict concrete projection coverage while excluding genuinely abstract classification families using their existing abstract-method authority. Address the broader #384 consumer gate. --- tests/unit/test_runtime_slice_projection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_runtime_slice_projection.py b/tests/unit/test_runtime_slice_projection.py index 78aec1511f..a3cc39fdf8 100644 --- a/tests/unit/test_runtime_slice_projection.py +++ b/tests/unit/test_runtime_slice_projection.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +import inspect from typing import Any, get_args, get_origin import numpy as np @@ -85,6 +86,8 @@ def _annotation_leaf_types(annotation: object) -> tuple[type, ...]: def test_cellprofiler_runtime_parameter_types_declare_slice_projection() -> None: missing: list[str] = [] for parameter_type in _subclasses(KeywordRuntimeParameter): + if inspect.isabstract(parameter_type): + continue if not parameter_type.__module__.startswith( ("openhcs.processing.backends.cellprofiler", "openhcs.interop.cellprofiler") ): From 7e0da269643a5d51122ee556611f9dfa81fb80fe Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 23:11:30 -0400 Subject: [PATCH 036/125] Record complete ordinary runtime qualification and preserved guard evidence Cite #384 results with exact source and dependency pins, all exported science comparisons, current main synchronization and remaining whole-value runtime frontier. --- .../shared_runtime_plumbing_384_20261001.rst | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 4ed582c9e5..35c5379c89 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -485,3 +485,110 @@ Evidence: ``registry-startup-readiness-162-empty-cache-pipeline-20261002.json``, ``registry-startup-readiness-162-original-r1-initialized-root-20261002.log``. The original 50.800-second compilation regression remains retained. Current combined performance, IFC behavior and source guards remain draft gates. + +Complete-field ordinary ABBA and current ownership repairs +--------------------------------------------------------- + +The completed uninstrumented ABBA at main107498cf against candidate02609e12 +has two observations per source for each workload. Execution/total means are +11.162969/13.119510 s versus 8.295599/10.351318 s for 3D, +8.706567/11.262818 s versus 8.474416/11.132766 s for Advanced, and +16.500067/18.794964 s versus 16.920803/19.454312 s for IFC. Thus 3D saves +2.867370 s execution and 2.768192 s total, while IFC regresses 0.420735 s +execution and 0.659348 s total in these samples. No statistical significance +or noise explanation is asserted. All observations remain retained. + +Original unmodified R0/R1 rejected02609e12. The actual metadata owner now owns +normalization and leading-axis transformation; the redundant raw projected +field dictionary, provenance normalization forwarder and foreign optional-state +handling are removed. Source and axis projection roles compose through +inheritance while preserving effect/error order and independent result ownership. +Common scalar admission is shared on SourceMetadataFields. The optional +intensity-proof projection and scale queries consume their existing authorities +instead of duplicating algorithms. Component controls include 41 scalar tests, +1,212 original-source behavior comparisons and 197 metadata/proof controls. +The collected classification projection census was also repaired to skip actually +abstract parameter declarations while still checking concrete descendants; +429 focused and 1,111 broader consumer controls passed before dependency update. +The inherited census failure was reproduced on main under the same imports. + +The subsequent ordinary ABBA measures frozen production candidate +90c30e1adfacd25a834ed48949b430099c866134 against the same main107498cf: + +==================== ======================== ======================== +Workload Execution main/candidate Total main/candidate +==================== ======================== ======================== +3D 11.089386 / 8.172661 s 13.063124 / 10.106094 s +Advanced 8.854658 / 8.988559 s 11.408556 / 11.590607 s +IFC 17.382926 / 16.212604 s 19.628127 / 18.661190 s +==================== ======================== ======================== + +3D saves 2.916725 s execution and 2.957031 s total; compilation means are +1.214262 / 1.209572 s. Advanced regresses 0.133900 s execution and 0.182051 s +total. IFC improves 1.170322 s execution and 0.966938 s total in this batch. +These means do not erase the adverse026 IFC evidence or establish significance. +All twelve case observations succeed through ordinary OUTCOMES, default memory +observation, one well/thread and CPU5. Both sources use identical public drivers, +shared dependencies/native binaries and mandatory registry preparation before +readiness. Server startup, prewarming and shutdown are excluded. There are no +profiling hooks, captures or scientific substitutions in either ABBA. + +For each ABBA, all six output pairs pass scientific comparison. All six 3D +CSV exports and all120TIFF arrays match exactly, including complete scientific +inventories, dtype and shape. IFC has exact headers and all1,800x527 result cells. +Advanced uses the existing CellProfiler database/export comparator at absolute +and relative1e-6, with exact schemas, cardinalities, discrete values, identifier +relationships and categorical data. Eight scientific subject tables and all +relationships are nonempty. Unsaved final segmentation masks were not observed. +The90 comparison reads actualaf45 source against its still unchanged pre-update +dependency environment, attested before and after. Output files remain unchanged. + +At90c30, original R1 passes with no increases under the unchanged160-second +budget, and original R0 scripts/benchmark pass. R0 openhcs remains RED solely +because the scalar/container rejection classifier moved from virtual workspace +decoding to its actual field owner: source_metadata gains one local switch and +three arms, while virtual_workspace_metadata loses one switch and four arms. +The original per-file gate is not waived or declared passed. Across all changed +roots the global delta is zero switches and minus one arm; untouched files cancel +because these original metrics are file-local AST measurements. Existing error +and reported-class observation ordering is preserved rather than altered to +satisfy the metric. The global702-module authority audit found no existing +replacement retaining that complete grammar/error contract. These are distinct +architecture and behavioral gates, not interchangeable claims. + +After both ABBA and all their comparisons finished, main +c32447f1c86a1878a313d1643a398e30ac20f75e was normally merged into this branch. +The shared environment now installs declared published ArrayBridge0.3.6 at +source1e53d03d9f468322a1c085c8be29485fb139caf2, openhcs-basicpy1.3.1 and +JAX/jaxlib0.9.2. Metaclass-registry0.2.2 source393a7e0, NumPy/SciPy/Numba +and both native extension hashes are unchanged. Pip check reports no broken +requirements and actual package imports pass. The separate native CellProfiler +4.2.8.1 environment remains unchanged. Earlier timings retain their measured +source/dependency pins; they are not projected through this new main/environment. +New combined qualification and full native parity remain required. + +A capture-free coarse diagnostic on frozen80d92 source partitions one9.208835 s +job into4.164762 s inside RuntimeCallableInvocation.call and5.044073 s outside. +The callable boundary includes decorated processing/metadata and is not a pure +image-kernel measurement. Diagnostic wrappers are not accepted performance +comparisons. Full load, unstack, save, CP image recording and image request phases +have a nonoverlapping3.009640 s upper bound. Recovering2 s requires eliminating +at least66.453% of those complete phases, not just a cached recomposition or +an original-source cache miss. The existing named-value to eager-plane metadata +to MemoryVFS to whole-stack round trip is the next structural premise. Mandatory +named/mainflow pixel-copy isolation and concurrent durable publication remain +constraints. No new production route is admitted without a representative replay +showing that complete consumer closure can realize a material payoff. + +Retained evidence is in the shared benchmark-runs directory: +``owned-metadata-90c30-ordinary-abba-csv-tiff-parity-20261002.json``, +``advanced-ordinary90-six-pair-sqlite-parity-20261002.json``, +``pr394-integrated-90c30-original-source-guards-20261002.json``, +``scalar-owner-original-r0-global-delta-counterevidence-20261002.json``, +``integrated90-classification-concrete-projection-census-fix-20261002.json``, +``shared-main-dependency-update-c32447-20261002.json`` and +``pr394-90c30-whole-mainflow-value-frontier-20261002.json``. +Source freezes, complete observations and diagnostic boundaries are retained in +``/var/tmp/openhcs-owned-metadata-90c30-ordinary-abba-20261002/`` and +``/var/tmp/openhcs-owned-metadata-phase-probe-20261002/``. The original026 +comparison and original failures remain separate pinned evidence. From 4afd5141554299680fa0326b31334a448fbf20bf Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 23:14:32 -0400 Subject: [PATCH 037/125] Clarify pinned runtime validation prose --- .../shared_runtime_plumbing_384_20261001.rst | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 35c5379c89..c6181ca111 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -489,7 +489,7 @@ combined performance, IFC behavior and source guards remain draft gates. Complete-field ordinary ABBA and current ownership repairs --------------------------------------------------------- -The completed uninstrumented ABBA at main107498cf against candidate02609e12 +The completed uninstrumented ABBA at main 107498cf against candidate 02609e12 has two observations per source for each workload. Execution/total means are 11.162969/13.119510 s versus 8.295599/10.351318 s for 3D, 8.706567/11.262818 s versus 8.474416/11.132766 s for Advanced, and @@ -498,7 +498,7 @@ has two observations per source for each workload. Execution/total means are execution and 0.659348 s total in these samples. No statistical significance or noise explanation is asserted. All observations remain retained. -Original unmodified R0/R1 rejected02609e12. The actual metadata owner now owns +Original unmodified R0/R1 rejected 02609e12. The actual metadata owner now owns normalization and leading-axis transformation; the redundant raw projected field dictionary, provenance normalization forwarder and foreign optional-state handling are removed. Source and axis projection roles compose through @@ -513,7 +513,7 @@ abstract parameter declarations while still checking concrete descendants; The inherited census failure was reproduced on main under the same imports. The subsequent ordinary ABBA measures frozen production candidate -90c30e1adfacd25a834ed48949b430099c866134 against the same main107498cf: +90c30e1adfacd25a834ed48949b430099c866134 against the same main 107498cf: ==================== ======================== ======================== Workload Execution main/candidate Total main/candidate @@ -534,16 +534,16 @@ readiness. Server startup, prewarming and shutdown are excluded. There are no profiling hooks, captures or scientific substitutions in either ABBA. For each ABBA, all six output pairs pass scientific comparison. All six 3D -CSV exports and all120TIFF arrays match exactly, including complete scientific -inventories, dtype and shape. IFC has exact headers and all1,800x527 result cells. +CSV exports and all 120 TIFF arrays match exactly, including complete scientific +inventories, dtype and shape. IFC has exact headers and all 1,800 x 527 result cells. Advanced uses the existing CellProfiler database/export comparator at absolute -and relative1e-6, with exact schemas, cardinalities, discrete values, identifier +and relative 1e-6, with exact schemas, cardinalities, discrete values, identifier relationships and categorical data. Eight scientific subject tables and all relationships are nonempty. Unsaved final segmentation masks were not observed. The90 comparison reads actualaf45 source against its still unchanged pre-update dependency environment, attested before and after. Output files remain unchanged. -At90c30, original R1 passes with no increases under the unchanged160-second +At 90c30, original R1 passes with no increases under the unchanged 160-second budget, and original R0 scripts/benchmark pass. R0 openhcs remains RED solely because the scalar/container rejection classifier moved from virtual workspace decoding to its actual field owner: source_metadata gains one local switch and @@ -552,28 +552,28 @@ The original per-file gate is not waived or declared passed. Across all changed roots the global delta is zero switches and minus one arm; untouched files cancel because these original metrics are file-local AST measurements. Existing error and reported-class observation ordering is preserved rather than altered to -satisfy the metric. The global702-module authority audit found no existing +satisfy the metric. The global 702-module authority audit found no existing replacement retaining that complete grammar/error contract. These are distinct architecture and behavioral gates, not interchangeable claims. After both ABBA and all their comparisons finished, main c32447f1c86a1878a313d1643a398e30ac20f75e was normally merged into this branch. -The shared environment now installs declared published ArrayBridge0.3.6 at -source1e53d03d9f468322a1c085c8be29485fb139caf2, openhcs-basicpy1.3.1 and -JAX/jaxlib0.9.2. Metaclass-registry0.2.2 source393a7e0, NumPy/SciPy/Numba +The shared environment now installs declared published ArrayBridge 0.3.6 at +source 1e53d03d9f468322a1c085c8be29485fb139caf2, openhcs-basicpy 1.3.1 and +JAX/jaxlib 0.9.2. Metaclass-registry 0.2.2 source 393a7e0, NumPy/SciPy/Numba and both native extension hashes are unchanged. Pip check reports no broken requirements and actual package imports pass. The separate native CellProfiler 4.2.8.1 environment remains unchanged. Earlier timings retain their measured source/dependency pins; they are not projected through this new main/environment. New combined qualification and full native parity remain required. -A capture-free coarse diagnostic on frozen80d92 source partitions one9.208835 s -job into4.164762 s inside RuntimeCallableInvocation.call and5.044073 s outside. +A capture-free coarse diagnostic on frozen 80d92 source partitions one 9.208835 s +job into 4.164762 s inside RuntimeCallableInvocation.call and 5.044073 s outside. The callable boundary includes decorated processing/metadata and is not a pure image-kernel measurement. Diagnostic wrappers are not accepted performance comparisons. Full load, unstack, save, CP image recording and image request phases -have a nonoverlapping3.009640 s upper bound. Recovering2 s requires eliminating -at least66.453% of those complete phases, not just a cached recomposition or +have a nonoverlapping 3.009640 s upper bound. Recovering 2 s requires eliminating +at least 66.453% of those complete phases, not just a cached recomposition or an original-source cache miss. The existing named-value to eager-plane metadata to MemoryVFS to whole-stack round trip is the next structural premise. Mandatory named/mainflow pixel-copy isolation and concurrent durable publication remain @@ -590,5 +590,5 @@ Retained evidence is in the shared benchmark-runs directory: ``pr394-90c30-whole-mainflow-value-frontier-20261002.json``. Source freezes, complete observations and diagnostic boundaries are retained in ``/var/tmp/openhcs-owned-metadata-90c30-ordinary-abba-20261002/`` and -``/var/tmp/openhcs-owned-metadata-phase-probe-20261002/``. The original026 +``/var/tmp/openhcs-owned-metadata-phase-probe-20261002/``. The original 026 comparison and original failures remain separate pinned evidence. From 82243634ce097d0d1a2d9e77664cbfeac3e6ec94 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 1 Oct 2026 23:52:10 -0400 Subject: [PATCH 038/125] Publish every successfully saved result destination through output owners (#134) --- ...sisted_result_publication_134_20261002.rst | 47 +++ openhcs/core/steps/function_outputs.py | 52 ++- openhcs/core/virtual_workspace_metadata.py | 13 +- tests/unit/test_function_outputs.py | 17 +- .../test_persisted_roi_only_publication.py | 313 ++++++++++++++++++ 5 files changed, 416 insertions(+), 26 deletions(-) create mode 100644 docs/validation/persisted_result_publication_134_20261002.rst create mode 100644 tests/unit/test_persisted_roi_only_publication.py diff --git a/docs/validation/persisted_result_publication_134_20261002.rst b/docs/validation/persisted_result_publication_134_20261002.rst new file mode 100644 index 0000000000..fe8e061086 --- /dev/null +++ b/docs/validation/persisted_result_publication_134_20261002.rst @@ -0,0 +1,47 @@ +Saved-result publication through the existing output owners +========================================================== + +PR394 integrates the source proposal supplied by PR404 (issue134), against +shared runtime source4afd51415. Successfully saved result directories participate +in publication even when their files are not raster images. This changes the +shared publication contract, not an ROI-specific writer or a function case. + +Existing OutputTarget owns guarded contains_outputs and a stored_output_paths +hook. RuntimeArtifactMetadataTarget overrides that hook for all saved formats; +no additional capability class is needed for its sole existing consumer. +The original MaterializationBatch saved outcomes determine runtime destinations. +Publication cannot rerender a mutable batch or invent an outcome from unrelated +filesystem files. Exact plate-relative result-directory ancestry is retained. +Final reconciliation derives previously declared directories from the existing +OpenHCSMetadataHandler after runtime values have been released. + +AtomicMetadataWriter remains the single transaction. It constructs image +component projections only for nonempty addressed image sets. SourceProjectionSet +retains its nonempty invariant, and actual saved images without produced source +addresses still fail without changing the metadata document. Result-only +publication produces an empty image inventory and projection, not an invented +image or source address. Existing format inventory remains authoritative. + +The current-source original control reproduces four failures and one pass. +The actual two-owner implementation passes89 publication, checkpoint, metadata +and materialization-journey tests. Five source cases exercise real bundle writer +saving, simple and nested result-only directories, original public result queries, +reconciliation after release, byte-exact refusal of unaddressed images, and a +new independently declared target with cooperative publication hooks. Its test +declaration explicitly requests its own registry key and restores the complete +registry after use; the initially inherited-key test contamination is retained. +The existing NumPy export control now requires a declared result destination +with empty raster inventory, instead of incorrectly requiring no metadata file. + +The NRA owner inventory joins137 original declarations across the selected +existing owners, scans1,498 caller modules without parse failures and records +alternative owners and open binding leads. Those syntax facts are not proof of +runtime binding or an architecture-gate pass. Original-source failure, failed +initial test path, registry-contamination failure, corrected consumer gate and +inventory receipts remain in the shared benchmark-runs workspace under +pr394-roi-publication prefixes. + +This is saved-output admission correctness and establishes no performance gain. +Synthetic archive bytes do not establish native ROI geometry, scientific pixels, +source provenance reopening or complete issue134 graph acceptance. Those broader +PR404 gates remain open. No additional scientific pipeline is executed here. diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py index 05b7fb539c..d8f615c97e 100644 --- a/openhcs/core/steps/function_outputs.py +++ b/openhcs/core/steps/function_outputs.py @@ -781,19 +781,20 @@ def runtime_artifact_projection_paths( return self.project_runtime_artifacts(context, plan) return () - def contains_images(self, context: ProcessingContext) -> bool: - """Return whether the completed target contains image outputs.""" + def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: + """Return this declaration's outputs eligible for publication.""" + return tuple( + context.filemanager.list_image_files(self.output_dir, self.backend) + ) + + def contains_outputs(self, context: ProcessingContext) -> bool: + """Return whether this declared destination has publishable outputs.""" if context.filemanager is None: raise ValueError("OpenHCS metadata requires a file manager.") if not context.filemanager.is_dir(self.output_dir, self.backend): return False - return bool( - context.filemanager.list_image_files( - self.output_dir, - self.backend, - ) - ) + return bool(self.stored_output_paths(context)) def write( self, @@ -832,7 +833,10 @@ def write( component_labels=context.metadata_cache, backend=self.backend, is_main=self.is_main, - results_dir=(Path(self.results_dir).name if self.results_dir else None), + results_dir=( + str(Path(self.results_dir).relative_to(self.plate_root)) + if self.results_dir is not None else None + ), ) def produced_projection_metadata( @@ -1063,7 +1067,7 @@ def finalize_completed_plate( for owner, context in target_contexts.items(): for target in owner.reconciliation_targets(context): - if target.contains_images(context): + if target.contains_outputs(context): target.write(context) @@ -1146,7 +1150,11 @@ def from_plan( class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): - """Image artifacts persist independently of main-flow image/checkpoint storage.""" + """Saved artifacts own result destinations independently of image storage.""" + + def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: + """Include every saved format in this declared result destination.""" + return tuple(context.filemanager.list_files(self.output_dir, self.backend)) def production_targets( self, @@ -1158,19 +1166,20 @@ def production_targets( Path(output.path).parent for materialization in self.artifact_materializations for output in materialization.outputs_for_backend(self.backend) - if ImageFileFormat.is_image_path(output.path) ) return tuple( target for directory in directories for target in (self.for_directory(directory),) - if target.contains_images(context) + if target.contains_outputs(context) ) def reconciliation_targets( self, context: ProcessingContext ) -> tuple[RuntimeArtifactMetadataTarget, ...]: """Use durable typed projections, without reloading cleaned artifact values.""" + from openhcs.microscopes.openhcs import OpenHCSMetadataHandler + subdirectories = OpenHCSMetadataSubdirectories.from_path( METADATA_CONFIG.metadata_path(self.plate_root) ) @@ -1183,15 +1192,26 @@ def reconciliation_targets( if isinstance(projection, SourceArtifactProjection) and projection.ref.backend == self.backend ) + result_directories = OpenHCSMetadataHandler( + context.filemanager + ).analysis_result_directories(self.plate_root) return tuple( self.for_directory(directory) - for directory in dict.fromkeys((self.output_dir, *directories)) + for directory in dict.fromkeys( + ( + self.output_dir, + *directories, + *(directory.path for directory in result_directories), + ) + ) ) def for_directory(self, directory: Path) -> RuntimeArtifactMetadataTarget: """Retain the compiled plate/backend while projecting one declared directory.""" sub_dir = directory.relative_to(self.plate_root) - return replace(self, output_dir=directory, sub_dir=str(sub_dir)) + return replace( + self, output_dir=directory, sub_dir=str(sub_dir), results_dir=str(directory) + ) @classmethod def from_plan(cls, plan: CompiledStepPlan) -> RuntimeArtifactMetadataTarget | None: @@ -1205,7 +1225,7 @@ def from_plan(cls, plan: CompiledStepPlan) -> RuntimeArtifactMetadataTarget | No backend=materialization.require_persistent_backend(), plate_root=plate_root, sub_dir=str(output_dir.relative_to(plate_root)), - results_dir=None, + results_dir=str(output_dir), ) diff --git a/openhcs/core/virtual_workspace_metadata.py b/openhcs/core/virtual_workspace_metadata.py index ded7fe31be..fc0c25cee3 100644 --- a/openhcs/core/virtual_workspace_metadata.py +++ b/openhcs/core/virtual_workspace_metadata.py @@ -232,12 +232,13 @@ def update(data): ) ) subdirectory.update(serializer.projection_fields(retained_paths)) - projections = SourceProjectionSet( - tuple(entries[path] for path in published_paths) - ) - subdirectory.update( - serializer.component_metadata(projections, labels=component_labels) - ) + if published_paths: + projections = SourceProjectionSet( + tuple(entries[path] for path in published_paths) + ) + subdirectory.update( + serializer.component_metadata(projections, labels=component_labels) + ) subdirectory[FIELDS.IMAGE_FILES] = list(published_paths) subdirectory[FIELDS.MICROSCOPE_HANDLER_NAME] = microscope_handler_name subdirectory[FIELDS.SOURCE_FILENAME_PARSER_NAME] = ( diff --git a/tests/unit/test_function_outputs.py b/tests/unit/test_function_outputs.py index 7d887b0825..98d2cad9f9 100644 --- a/tests/unit/test_function_outputs.py +++ b/tests/unit/test_function_outputs.py @@ -2490,15 +2490,24 @@ def test_actual_array_exports_only_publish_declared_raster_inventory( context.runtime_value_store.clear() OpenHCSMetadataWriter.finalize_completed_plate({"A01": context}) metadata_path = tmp_path / "openhcs_metadata.json" + subdirectories = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES] + assert set(subdirectories) == {"images/exports"} + subdirectory = subdirectories["images/exports"] + assert subdirectory[SourceProjectionMetadataSerializer.RESULTS_DIR_FIELD] == ( + "images/exports" + ) if not with_raster: - assert not metadata_path.exists() + assert subdirectory[FIELDS.IMAGE_FILES] == [] + assert ( + VirtualWorkspaceSourceProjectionEntries.from_subdirectory( + subdirectory + ).entries + == {} + ) return np.testing.assert_array_equal( tifffile.imread(tmp_path / "images/exports/A01_s001_w2_z001_t001.tif"), pixels ) - subdirectories = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES] - assert set(subdirectories) == {"images/exports"} - subdirectory = subdirectories["images/exports"] assert subdirectory[FIELDS.IMAGE_FILES] == [ "images/exports/A01_s001_w2_z001_t001.tif" ] diff --git a/tests/unit/test_persisted_roi_only_publication.py b/tests/unit/test_persisted_roi_only_publication.py new file mode 100644 index 0000000000..7338a41787 --- /dev/null +++ b/tests/unit/test_persisted_roi_only_publication.py @@ -0,0 +1,313 @@ +"""Saved-output publication only: synthetic bytes, no ROI or pixel decoding.""" + +from dataclasses import replace +from types import SimpleNamespace + +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +import json + +import pytest + +from openhcs.agent.dto.plate import PlateFileQueryRequest +from openhcs.agent.path_policy import AgentPathPolicy +from openhcs.agent.services.plate_inspection_service import ( + PlateInspectionFileManagerFactory, + PlateInspectionService, +) +from openhcs.constants.constants import Backend +from openhcs.core.artifacts import ArtifactOutputPlan, MetadataArtifactType +from openhcs.core.compiled_step_plan import ( + CompiledStepPlan, + RuntimeArtifactMaterializationPlan, +) +from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.runtime_artifact_values import RuntimeValue +from openhcs.core.runtime_stores import RuntimeArtifactLocation, StoredRuntimeValue +from openhcs.core.steps.function_artifact_materialization import ( + MaterializedRuntimeArtifact, + RuntimeArtifactMaterialization, +) +from openhcs.core.steps.function_outputs import ( + OpenHCSMetadataWriter, + ProducedImageMetadataCapability, + RuntimeArtifactMetadataTarget, +) +from openhcs.core.virtual_workspace_metadata import METADATA_CONFIG, MetadataWriteError +from openhcs.processing.materialization.core import ( + MaterializationSpec, + _WRITERS_BY_OPTIONS, + prepare_materialization, +) +from openhcs.processing.materialization.options import FileBundleOptions + + +@pytest.fixture +def publication_context(tmp_path): + filemanager = FileManager({Backend.DISK.value: DiskStorageBackend()}) + context = ProcessingContext(filemanager=filemanager) + context.metadata_cache = {} + context.microscope_handler = SimpleNamespace( + parser=SourceSchemaFilenameParser(), microscope_type="openhcsdata" + ) + context.publication_audit = [] + return context + + +class PublicationAuditCapability: + """Independent audit capability extends, rather than replaces, publication.""" + + def write(self, context, *, produced_plan=None): + context.publication_audit.append(("before", self.sub_dir)) + super().write(context, produced_plan=produced_plan) + context.publication_audit.append(("after", self.sub_dir)) + + +def plan_for(plate): + return CompiledStepPlan( + step_index=0, + step_name="synthetic saved result", + step_type="FunctionStep", + axis_id="A01", + write_backend=Backend.MEMORY.value, + create_openhcs_metadata=True, + output_plate_root=str(plate), + analysis_results_dir=str(plate / "compiled_default"), + runtime_artifact_materialization=RuntimeArtifactMaterializationPlan( + persistent_enabled=True, + persistent_backend=Backend.DISK.value, + ), + ) + + +def save_archive_outcome(context, plate, destination, monkeypatch): + """Use the original declared bundle writer, batch and disk save boundary.""" + spec = MaterializationSpec(FileBundleOptions()) + payload = { + f"{destination}/independent-outline.roi.zip": b"synthetic admission only" + } + base_path = plate / "bundle" + batch = prepare_materialization( + spec, + payload, + str(base_path), + context.filemanager, + (Backend.DISK.value,), + ) + saved = batch.save() + (output,) = saved.outputs_for_backend(Backend.DISK.value) + assert output is batch.outputs[0] + # Source qualification must consume the original successful save, not render + # a mutable logical payload again or use planned names as proof of a save. + payload.clear() + + def forbidden_render(*args, **kwargs): + pytest.fail("Publication rendered the already-saved batch again") + + writer = _WRITERS_BY_OPTIONS[FileBundleOptions] + monkeypatch.setitem( + _WRITERS_BY_OPTIONS, FileBundleOptions, replace(writer, write=forbidden_render) + ) + artifact_plan = ArtifactOutputPlan( + name="synthetic archive bundle", + path="/memory/bundle.pkl", + artifact_type=MetadataArtifactType, + materialization=spec, + ) + record = StoredRuntimeValue( + value=RuntimeValue.from_output_plan( + artifact_plan, + {}, + execution_scope=RuntimeExecutionAxisScope(axis_id="A01"), + ), + location=RuntimeArtifactLocation( + path=artifact_plan.path, backend=Backend.MEMORY.value + ), + ) + materialization = RuntimeArtifactMaterialization( + output_plan=artifact_plan, + spec=spec, + record=record, + data=payload, + base_path=base_path, + source_identity=None, + filename_source_identity=None, + ) + return ( + MaterializedRuntimeArtifact( + outputs_by_backend=saved.outputs_by_backend, + materialization=materialization, + ), + output, + ) + + +class ExistingDiskInspectionFactory(PlateInspectionFileManagerFactory): + """Original public service receives the same real disk manager as saving.""" + + def __init__(self, manager): + self.manager = manager + + def create(self): + return self.manager + + +@pytest.fixture(autouse=True) +def forbid_optional_backend_bootstrap(monkeypatch): + import polystore.base + + def forbidden_bootstrap(*args, **kwargs): + pytest.fail("Source-only qualification must not bootstrap optional backends") + + monkeypatch.setattr(polystore.base, "ensure_storage_registry", forbidden_bootstrap) + + +def result_paths(plate, context): + service = PlateInspectionService( + AgentPathPolicy.with_roots(readable_roots=(plate,), writable_roots=()), + filemanager_factory=ExistingDiskInspectionFactory(context.filemanager), + ) + result = service.query_files( + PlateFileQueryRequest.from_fields( + plate_path=str(plate), + microscope_type="openhcsdata", + kind="result", + include_previews=False, + ) + ) + assert result.errors == () + assert all(record.preview is None for record in result.records) + return tuple(record.full_path for record in result.records) + + +@pytest.mark.parametrize("destination", ("roi_only", "independent/nested/roi_only")) +def test_saved_roi_only_batch_publishes_and_reconciles_without_rendering( + tmp_path, + publication_context, + monkeypatch, + destination, +): + plate = tmp_path / "plate" + outcome, output = save_archive_outcome( + publication_context, plate, destination, monkeypatch + ) + plan = plan_for(plate) + targets = OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, + plan, + artifact_materializations=(outcome,), + ) + assert tuple(target.output_dir for target in targets) == (plate / destination,) + OpenHCSMetadataWriter.write( + publication_context, plan, artifact_materializations=(outcome,) + ) + assert result_paths(plate, publication_context) == (output.path,) + published = json.loads(METADATA_CONFIG.metadata_path(plate).read_text()) + assert published["subdirectories"][destination]["image_files"] == [] + assert published["subdirectories"][destination]["source_projection"] == [] + # Final reconciliation has no runtime outcome/value to reconstruct. Its + # destination comes from the same durable declaration the reader admitted. + publication_context.step_plans = {0: plan} + owner = RuntimeArtifactMetadataTarget.from_plan(plan) + reconciled = tuple( + target.output_dir + for target in owner.reconciliation_targets(publication_context) + ) + assert plate / destination in reconciled + OpenHCSMetadataWriter.finalize_completed_plate({"A01": publication_context}) + assert result_paths(plate, publication_context) == (output.path,) + + +def test_shared_writer_accepts_result_only_but_not_unaddressed_images( + tmp_path, + publication_context, +): + plate = tmp_path / "plate" + directory = plate / "standalone_result" + directory.mkdir(parents=True) + roi = directory / "independent-outline.roi.zip" + roi.write_bytes(b"synthetic admission only") + target = RuntimeArtifactMetadataTarget.from_plan(plan_for(plate)).for_directory( + directory + ) + target.write(publication_context) + assert result_paths(plate, publication_context) == (str(roi),) + metadata_path = METADATA_CONFIG.metadata_path(plate) + before = metadata_path.read_bytes() + (directory / "unaddressed-image.tif").write_bytes(b"never decoded") + with pytest.raises( + MetadataWriteError, match="Saved images lack typed produced addresses" + ): + target.write(publication_context) + assert metadata_path.read_bytes() == before + + +def test_new_result_declaration_composes_cooperative_hooks_without_consumer_edits( + tmp_path, + publication_context, + monkeypatch, +): + plate = tmp_path / "plate" + destination = "new/declaration_only" + outcome, output = save_archive_outcome( + publication_context, plate, destination, monkeypatch + ) + registry = OpenHCSMetadataWriter.OutputTarget.__registry__ + original_declarations = dict(registry) + try: + + class IndependentSavedResultMetadataTarget( + PublicationAuditCapability, + ProducedImageMetadataCapability, + RuntimeArtifactMetadataTarget, + ): + declaration_key = None + + @classmethod + def from_plan(cls, plan): + return cls( + output_dir=plate / destination, + backend=Backend.DISK.value, + plate_root=str(plate), + sub_dir=destination, + results_dir=str(plate / destination), + ) + + plan = replace( + plan_for(plate), + runtime_artifact_materialization=RuntimeArtifactMaterializationPlan(), + ) + (target,) = OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, + plan, + artifact_materializations=(outcome,), + ) + target.write(publication_context, produced_plan=plan) + assert publication_context.publication_audit == [ + ("before", destination), + ("after", destination), + ] + assert result_paths(plate, publication_context) == (output.path,) + finally: + registry.clear() + registry.update(original_declarations) + + +def test_files_without_successful_batch_outcomes_do_not_create_step_targets( + tmp_path, + publication_context, +): + plate = tmp_path / "plate" + directory = plate / "unclaimed" + directory.mkdir(parents=True) + (directory / "not_a_saved_outcome.roi.zip").write_bytes(b"synthetic admission only") + assert ( + OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, + plan_for(plate), + artifact_materializations=(), + ) + == () + ) From 77a5f0a4b2f1116a988e79d675de2a8b5ae61864 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 01:06:16 -0400 Subject: [PATCH 039/125] Prepare canonical and raw callable contracts before readiness and compilation (#419) --- ...pared_callable_signatures_419_20261002.rst | 54 +++ openhcs/core/callable_contract.py | 267 ++++++++--- openhcs/core/function_contract_metadata.py | 2 + openhcs/core/function_patterns.py | 2 +- openhcs/core/pipeline/function_contracts.py | 17 +- openhcs/core/runtime_batch_contracts.py | 13 +- openhcs/core/steps/function_runtime.py | 4 - .../runtime/function_contract_execution.py | 63 ++- .../backends/cellprofiler/classification.py | 2 +- .../backends/cellprofiler/colocalization.py | 8 +- .../processing/backends/cellprofiler/color.py | 26 +- .../backends/cellprofiler/display_modules.py | 3 +- .../processing/backends/cellprofiler/edge.py | 3 +- .../cellprofiler/feature_enhancement.py | 3 +- .../backends/cellprofiler/gaussian_filter.py | 3 +- .../backends/cellprofiler/illumination.py | 2 +- .../backends/cellprofiler/image_geometry.py | 10 +- .../backends/cellprofiler/intensity.py | 3 +- .../cellprofiler/intensity_distribution.py | 4 +- .../backends/cellprofiler/median_filter.py | 3 +- .../backends/cellprofiler/morphology.py | 7 +- .../backends/cellprofiler/outlines.py | 10 +- .../backends/cellprofiler/primary_objects.py | 3 +- .../backends/cellprofiler/secondary.py | 7 +- .../backends/cellprofiler/skeleton.py | 2 +- .../backends/cellprofiler/smoothing.py | 7 +- .../processing/backends/cellprofiler/worms.py | 2 +- .../backends/lib_registry/registry_service.py | 9 +- .../backends/lib_registry/unified_registry.py | 35 +- tests/unit/test_callable_abi_preparation.py | 306 ++++++++++++ ...ellprofiler_function_contract_execution.py | 444 +++++++++++++++++- .../test_cellprofiler_prepared_payload_abi.py | 127 +++++ tests/unit/test_preparation_admission.py | 4 +- 33 files changed, 1293 insertions(+), 162 deletions(-) create mode 100644 docs/validation/prepared_callable_signatures_419_20261002.rst create mode 100644 tests/unit/test_callable_abi_preparation.py create mode 100644 tests/unit/test_cellprofiler_prepared_payload_abi.py diff --git a/docs/validation/prepared_callable_signatures_419_20261002.rst b/docs/validation/prepared_callable_signatures_419_20261002.rst new file mode 100644 index 0000000000..e987dab3b9 --- /dev/null +++ b/docs/validation/prepared_callable_signatures_419_20261002.rst @@ -0,0 +1,54 @@ +Prepared callable signatures and raw argument admission +====================================================== + +Registered argument contracts belong in function-library warmup before server +readiness. Newly authored declarations prepare before compiler validation and +resolve once per compilation. Invocation must consume those prepared contracts. +This is a correctness/preparation dependency of the plumbing work, not a claim +that signature caching alone removes multiple seconds. + +CallableMetadata carries the resolved canonical inspect.Signature. Parameter +names, annotations, defaults and semantic controls derive from that declaration. +Preparation publishes it only after every registered preparation hook finishes. +FunctionReference and existing namespace/pickle/cloudpickle transport retain it. +The compiler reuses registered snapshots and captures fresh authored snapshots; +the old late invocation preparation is removed. + +Actual boundary inspection found that most CP raw targets differ from their +semantic signatures because injected controls are absent from the raw function. +The existing metadata owner therefore also carries a raw-runtime Signature only +when required by that distinct boundary. Identical boundaries reuse canonical; +default objects are retained by identity and never compared through value equality. +Existing invocation and batch-request owners receive the exact raw view for +filtering/defaults. Three independent signature/default/type LRUs and the obsolete +signature-parameter helper are removed. Unprepared authoring and distinct generic +wrapper queries remain live. Explicit library rewarm or supported declaration +invalidation refreshes registered snapshots; arbitrary direct annotation/default +mutation does not retroactively rewrite compiled contracts. + +Raw array admission is derived from the callable's declared nominal type, including +unions and Annotated. The adapter retains context until the actual raw boundary. +Array-only callables receive pixels; declared RuntimeArrayData consumers retain +metadata. Their truthful annotation migration changes no scientific function body, +defaults or decorators, as checked across all 19 affected CP modules by AST. + +Controls: 646 passed, 5 existing skips, 2 existing warnings. They cover preparation +ordering, rewarm/new compilation, request bindings, custom signatures/defaults, +transport, and 12-plane compiled execution with all signature/hint queries forbidden. +An additional 75 publication/real-payload controls pass, including actual compiled +ColorToGray spatial-mask/channel collapse, MedianFilter intensity scale/mask and +IdentifySecondaryObjects source coordinates/provenance. + +The final real isolated registry check prepares 267 catalog entries and 976 targets; +all 267 contracts have snapshots, with zero missing signatures and zero subsequent +owner signature/hint resolutions in contract queries/projections. Its 16.67s +cached-startup duration is diagnostic startup and excluded from pipeline clocks. +The earlier fresh-cache 91.01s startup check remains separately preserved. + +Evidence is retained under /home/ts/code/projects/openhcs-benchmark-runs: +issue419-shared-signature-closure-final-receipt-20261002.json, +issue419-actual-shared-signature-closure-readiness-20261002.json and +issue419-truthful-annotation-actual-source-consumers-20261002.json. Earlier failed +intermediates are retained. Current all-case scientific parity and ordinary +performance are separate pending gates; earlier 90c30 timings do not qualify this +new source or updated shared dependencies. diff --git a/openhcs/core/callable_contract.py b/openhcs/core/callable_contract.py index d6de5863a5..d462097d29 100644 --- a/openhcs/core/callable_contract.py +++ b/openhcs/core/callable_contract.py @@ -22,6 +22,7 @@ Mapping, TypeVar, cast, + get_args, get_type_hints, overload, ) @@ -33,6 +34,7 @@ coerce_enum_member, declared_enum_type, enum_member_type, + is_union_type, resolve_annotated, validate_annotation_value, ) @@ -292,9 +294,28 @@ class CallableMetadata: image_payload_consumption: ImagePayloadConsumption = ImagePayloadConsumption.NATURAL request_binding: "CallableRequestBinding | None" = None prepare: Callable[..., object] | None = None + canonical_signature: inspect.Signature | None = None + raw_runtime_signature: inspect.Signature | None = None primary_image_carrier_requirement: PrimaryImageCarrierRequirement | None = None primary_image_carrier_transition: PrimaryImageCarrierTransition | None = None + @staticmethod + def prepared_callable_signature(func: Callable[..., object]) -> inspect.Signature | None: + """Admit an owned snapshot only for its exact semantic callable boundary.""" + raw = getattr(func, FunctionContractAttribute.raw_processing_function, None) + if raw is not None and raw is not func: + return None + signature = getattr(func, FunctionContractAttribute.canonical_signature, None) + if signature is not None and not isinstance(signature, inspect.Signature): + raise TypeError("Prepared callable signature must be an inspect.Signature or None.") + return signature + + @classmethod + def callable_signature(cls, func: Callable[..., object]) -> inspect.Signature: + """Read the prepared exact ABI; unprepared wrapper/authoring views stay live.""" + signature = cls.prepared_callable_signature(func) + return inspect.signature(func) if signature is None else signature + @property def artifact_output_policy(self) -> type[ArtifactOutputPolicy]: """Project recording ownership from this callable's adapter declaration.""" @@ -378,13 +399,14 @@ def from_projection( runtime_adapter = runtime_adapter_spec_from_callable(projection.func) if runtime_adapter is not None and callable(projection.func): runtime_adapter.validate_callable_signature(projection.func) + declared_artifact_inputs = _artifact_specs_from_namespace( + namespace, projection.name, FunctionContractAttribute.artifact_inputs, + ) + signature = ( + cls.callable_signature(projection.func) if callable(projection.func) else None + ) artifact_inputs = _artifact_input_specs_from_projection( - projection, - _artifact_specs_from_namespace( - namespace, - projection.name, - FunctionContractAttribute.artifact_inputs, - ), + projection, declared_artifact_inputs, signature, ) return cls( input_memory_type=reader.optional_string( @@ -401,6 +423,7 @@ def from_projection( projection, reader, artifact_inputs, + signature, ), artifact_outputs=_artifact_specs_from_namespace( namespace, @@ -422,7 +445,7 @@ def from_projection( FunctionContractAttribute.allowed_group_by, ), runtime_adapter=runtime_adapter, - runtime_context_parameter=_runtime_context_parameter(projection, reader), + runtime_context_parameter=_runtime_context_parameter(projection, reader, signature), execution_scope=reader.optional_execution_scope( FunctionContractAttribute.execution_scope, ), @@ -457,6 +480,12 @@ def from_projection( prepare=reader.optional_callable( FunctionContractAttribute.processing_prepare ), + canonical_signature=reader.optional_signature( + FunctionContractAttribute.canonical_signature, + ), + raw_runtime_signature=reader.optional_signature( + FunctionContractAttribute.raw_runtime_signature, + ), ) def without_prepare(self) -> "CallableMetadata": @@ -476,6 +505,10 @@ def with_raw_processing_function( def as_namespace(self) -> dict[str, object]: """Project typed metadata into callable declaration keys.""" namespace: dict[str, object] = {} + if self.canonical_signature is not None: + namespace[FunctionContractAttribute.canonical_signature] = self.canonical_signature + if self.raw_runtime_signature is not None: + namespace[FunctionContractAttribute.raw_runtime_signature] = self.raw_runtime_signature if self.input_memory_type is not None: MemoryContractAttribute.INPUT.write(namespace, self.input_memory_type) if self.output_memory_type is not None: @@ -777,10 +810,7 @@ def config_bound_parameters(self) -> tuple[inspect.Parameter, ...]: from openhcs.core.config import runtime_config_parameter - signature = inspect.signature( - self.resolve_canonical_raw_callable(), - eval_str=True, - ) + signature = self.canonical_signature return tuple( normalized for parameter in signature.parameters.values() @@ -843,7 +873,7 @@ def declared_path_values( ) -> Mapping[str, tuple["PlatePathDeclaration", object]]: """Return authored or signature-default values for declared paths.""" - signature = inspect.signature(self.resolve_canonical_raw_callable()) + signature = self.canonical_signature values: dict[str, tuple["PlatePathDeclaration", object]] = {} for parameter_name, declaration in self.declared_path_parameters.items(): value = kwargs.get(parameter_name, signature.parameters[parameter_name].default) @@ -923,17 +953,49 @@ def require_processing_contract(self) -> "ProcessingContract": ) return processing_contract - def raw_main_flow_call_argument(self, source_payload: Any) -> Any: + def runtime_main_flow_call_argument(self, source_payload: Any) -> Any: """Project this callable's ABI without discarding an adapter's context.""" - from arraybridge import ArrayPayload - from openhcs.core.runtime_image_values import image_payload_data - if self.runtime_adapter is not None: return source_payload - annotation = self.canonical_parameter_annotations.get(self.primary_input_parameter_name) - if isinstance(annotation, type) and issubclass(annotation, ArrayPayload): - return source_payload - return image_payload_data(source_payload) + return self.raw_main_flow_call_argument(source_payload) + + def raw_main_flow_call_argument( + self, source_payload: Any, + ) -> Any: + """Project one image argument at the canonical raw callable boundary.""" + from openhcs.core.runtime_image_values import image_payload_data + + return ( + source_payload + if any( + isinstance(source_payload, argument_type) + for argument_type in self.raw_main_flow_argument_types() + ) + else image_payload_data(source_payload) + ) + + def raw_main_flow_argument_types(self) -> tuple[type, ...]: + """Derive the nominal argument view for one canonical invocation.""" + parameters = self.canonical_parameter_declarations + primary_name = self._primary_input_name(parameters) + annotation = ( + None if primary_name is None else parameters[primary_name].annotation + ) + return self._nominal_argument_types(annotation) + + @classmethod + def _nominal_argument_types(cls, annotation: object) -> tuple[type, ...]: + annotation = resolve_annotated(annotation) + if is_union_type(annotation): + return tuple( + argument_type for member in get_args(annotation) + for argument_type in cls._nominal_argument_types(member) + ) + return ( + (annotation,) + if annotation is not Any and isinstance(annotation, type) + else () + ) def main_flow_call_argument(self, source_payload: Any) -> Any: """Let the processing declaration retain context needed before raw calls.""" @@ -1001,8 +1063,12 @@ def artifact_specs(self) -> ArtifactSpecCollection: @property def primary_input_parameter_name(self) -> str | None: """FunctionStep input payload parameter declared by callable signature.""" - signature = inspect.signature(self.resolve_canonical_raw_callable()) - for parameter in signature.parameters.values(): + signature = self.canonical_signature + return self._primary_input_name(signature.parameters) + + @staticmethod + def _primary_input_name(parameters: Mapping[str, inspect.Parameter]) -> str | None: + for parameter in parameters.values(): if parameter.kind in ( inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD, @@ -1089,14 +1155,102 @@ def is_request_boundary(candidate: Callable[..., object]) -> bool: @property def canonical_parameter_annotations(self) -> Mapping[str, object]: """Derive annotations from the original semantic signature owner.""" - - raw_callable = self.resolve_canonical_raw_callable() - annotations = get_type_hints(raw_callable, include_extras=True) return { - name: annotations.get(name, parameter.annotation) - for name, parameter in inspect.signature(raw_callable).parameters.items() + name: parameter.annotation + for name, parameter in self.canonical_parameter_declarations.items() } + @property + def canonical_parameter_declarations(self) -> Mapping[str, inspect.Parameter]: + """Resolve signature and annotations together at their declaration owner.""" + return self.canonical_signature.parameters + + @property + def canonical_signature(self) -> inspect.Signature: + """Use the prepared declaration, or query an uncompiled authoring callable.""" + if self.metadata.canonical_signature is not None: + return self.metadata.canonical_signature + return self.resolve_signature(self.resolve_canonical_raw_callable()) + + def resolve_signature(self, raw_callable: Callable[..., object]) -> inspect.Signature: + """Resolve the semantic ABI once after declaration preparation.""" + signature = inspect.signature(raw_callable) + annotations = ( + self.request_binding.public_annotations_dict + if self.request_binding is not None + else get_type_hints(raw_callable, include_extras=True) + ) + return signature.replace( + parameters=tuple( + parameter.replace(annotation=annotations.get(name, parameter.annotation)) + for name, parameter in signature.parameters.items() + ), + return_annotation=annotations.get("return", signature.return_annotation), + ) + + def with_prepared_signature(self) -> "CallableContract": + """Capture the prepared ABI on the declaration transported to workers.""" + canonical = self.resolve_canonical_raw_callable() + signature = self.resolve_signature(canonical) + runtime = self.resolve_raw_runtime_callable() + runtime_signature = None + if runtime is not canonical: + candidate = self.resolve_signature(runtime) + canonical_parameters = tuple(signature.parameters.values()) + runtime_parameters = tuple(candidate.parameters.values()) + same_parameters = len(canonical_parameters) == len(runtime_parameters) and all( + left.name == right.name and left.kind is right.kind + and left.default is right.default and left.annotation is right.annotation + for left, right in zip(canonical_parameters, runtime_parameters) + ) + if not (same_parameters and signature.return_annotation is candidate.return_annotation): + runtime_signature = candidate + return dataclasses.replace( + self, + metadata=dataclasses.replace( + self.metadata, canonical_signature=signature, + raw_runtime_signature=runtime_signature, + ), + ) + + @property + def raw_runtime_signature(self) -> inspect.Signature: + """The exact unwrapped execution ABI, preserving request boundaries.""" + if self.metadata.raw_runtime_signature is not None: + return self.metadata.raw_runtime_signature + if self.metadata.canonical_signature is not None: + return self.metadata.canonical_signature + return self.resolve_signature(self.resolve_raw_runtime_callable()) + + @classmethod + def warm_canonical_signature(cls, func: Callable[..., object]) -> None: + """Publish one refreshed ABI after all library preparation hooks finish.""" + contract = cls.from_callable(func).with_prepared_signature() + _mutable_callable_namespace(func)[FunctionContractAttribute.canonical_signature] = ( + contract.metadata.canonical_signature + ) + namespace = _mutable_callable_namespace(func) + namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) + if contract.metadata.raw_runtime_signature is not None: + namespace[FunctionContractAttribute.raw_runtime_signature] = contract.metadata.raw_runtime_signature + + @classmethod + def from_prepared_callable(cls, func: Callable[..., object]) -> "CallableContract": + """Reuse library warmup; prepare authored declarations before validation.""" + projection = CallableProjection.from_callable(func) + if projection.namespace.get(FunctionContractAttribute.canonical_signature) is not None: + return cls.from_callable(func) + resolved = _resolve_declared_callable(func) + for target in CallableProjection.from_callable(resolved).prepare_targets(): + prepare_processing_callable(target) + prepared = cls.from_callable(resolved) + if prepared.metadata.canonical_signature is None: + prepared = prepared.with_prepared_signature() + return dataclasses.replace( + prepared, func=func, function_name=projection.name, + module_name=projection.module_name, + ) + def decode_public_kwargs(self, kwargs: Mapping[str, object]) -> dict[str, object]: """Descend declared scalar enums at authoring, before source projection.""" @@ -1123,7 +1277,7 @@ def validate_public_kwargs( raise TypeError( "CallableContract.validate_public_kwargs requires a mapping." ) - signature = inspect.signature(self.resolve_canonical_raw_callable()) + signature = self.canonical_signature runtime_owned_value = object() call_kwargs = dict(kwargs) runtime_loaded_parameters = frozenset(runtime_loaded_artifact_parameter_names) @@ -1194,15 +1348,19 @@ def validate_artifact_input_parameter_bindings(self) -> None: """Validate exact artifact occurrences against the normalized callable ABI.""" from openhcs.core.pipeline.function_contracts import ( - resolved_callable_parameter, special_input_parameter_accepts_sequence, ) - raw_callable = self.resolve_canonical_raw_callable() - artifact_parameters = { - parameter_name: resolved_callable_parameter(raw_callable, parameter_name) - for parameter_name in self.artifact_input_parameter_names - } + artifact_parameters = {} + parameters = self.canonical_parameter_declarations + for parameter_name in self.artifact_input_parameter_names: + parameter = parameters.get(parameter_name) + if parameter is None: + raise ValueError( + f"Callable {self.function_name!r} does not declare parameter " + f"{parameter_name!r}." + ) + artifact_parameters[parameter_name] = parameter specs_by_parameter: dict[str, list[ArtifactSpec]] = {} for spec in self.artifact_inputs: parameter_name = spec.parameter_name @@ -1419,6 +1577,8 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return func(**binding.implementation_kwargs(bound.arguments)) wrapper_namespace = _mutable_callable_namespace(wrapper) + wrapper_namespace.pop(FunctionContractAttribute.canonical_signature, None) + wrapper_namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) wrapper_namespace[FunctionContractAttribute.callable_request_binding] = binding wrapper_namespace["__signature__"] = public_signature return wrapper @@ -1473,6 +1633,8 @@ def attach_callable_contract_metadata( primary_image_carrier_transition: PrimaryImageCarrierTransition | None = None, ) -> None: """Attach OpenHCS callable metadata used by compiler/runtime phases.""" + _mutable_callable_namespace(func).pop(FunctionContractAttribute.canonical_signature, None) + _mutable_callable_namespace(func).pop(FunctionContractAttribute.raw_runtime_signature, None) if declared_processing_contract is not None: if ( not isinstance(declared_processing_contract, str) @@ -1649,9 +1811,10 @@ def attach_processing_prepare(func: Any, prepare: Any) -> None: f"got {type(prepare).__name__}." ) for target in CallableProjection.from_callable(func).prepare_targets(): - _mutable_callable_namespace(target)[ - FunctionContractAttribute.processing_prepare - ] = prepare + namespace = _mutable_callable_namespace(target) + namespace.pop(FunctionContractAttribute.canonical_signature, None) + namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) + namespace[FunctionContractAttribute.processing_prepare] = prepare def runtime_image_execution_mode( @@ -1813,6 +1976,7 @@ def prepare_targets(self) -> tuple[Any, ...]: def _runtime_context_parameter( projection: CallableProjection, reader: "CallableMetadataReader", + signature: inspect.Signature | None, ) -> str | None: declared = reader.optional_string( FunctionContractAttribute.runtime_context_parameter @@ -1821,19 +1985,8 @@ def _runtime_context_parameter( return declared from openhcs.core.context.processing_context import ProcessingContext - return _callable_signature_parameter( - projection.func, - ProcessingContext.require_parameter_name(), - ) - - -def _callable_signature_parameter(func: Any, parameter_name: str) -> str | None: - """Return a callable parameter name accepted by live signatures only.""" - if not callable(func): - return None - if parameter_name not in inspect.signature(func).parameters: - return None - return parameter_name + parameter_name = ProcessingContext.require_parameter_name() + return parameter_name if signature is not None and parameter_name in signature.parameters else None @dataclass(frozen=True, slots=True) @@ -1843,6 +1996,12 @@ class CallableMetadataReader: namespace: CallableNamespace function_name: str + def optional_signature(self, field_name: str) -> inspect.Signature | None: + value = self.namespace.get(field_name) + if value is not None and not isinstance(value, inspect.Signature): + raise TypeError(f"{self.function_name!r}.{field_name} must be an inspect.Signature or None.") + return value + def optional_string(self, field_name: str) -> str | None: """Return an optional string metadata field.""" value = self.namespace.get(field_name) @@ -2193,12 +2352,12 @@ def _artifact_input_parameter_names( def _artifact_input_specs_from_projection( projection: CallableProjection, artifact_inputs: tuple[ArtifactSpec, ...], + signature: inspect.Signature | None, ) -> tuple[ArtifactSpec, ...]: """Bind exact same-name input declarations at the metadata boundary.""" - if not callable(projection.func): + if signature is None: return artifact_inputs - signature = inspect.signature(projection.func) primary_parameter_name = next( ( parameter.name @@ -2228,6 +2387,7 @@ def _artifact_input_parameter_names_from_projection( projection: CallableProjection, reader: CallableMetadataReader, artifact_inputs: tuple[ArtifactSpec, ...], + signature: inspect.Signature | None, ) -> tuple[str, ...]: """Normalize compatibility and exact artifact parameter declarations once.""" @@ -2281,9 +2441,8 @@ def _artifact_input_parameter_names_from_projection( f"disagree: {declarations}." ) - if not callable(projection.func): + if signature is None: return first_names - signature = inspect.signature(projection.func) ordered_names = tuple(name for name in signature.parameters if name in first_set) return ( *ordered_names, diff --git a/openhcs/core/function_contract_metadata.py b/openhcs/core/function_contract_metadata.py index b269577cb8..09713a46ce 100644 --- a/openhcs/core/function_contract_metadata.py +++ b/openhcs/core/function_contract_metadata.py @@ -16,6 +16,8 @@ class FunctionContractAttribute: "__openhcs_declared_processing_contract__" ) raw_processing_function: ClassVar[str] = "__openhcs_raw_processing_function__" + canonical_signature: ClassVar[str] = "__openhcs_canonical_signature__" + raw_runtime_signature: ClassVar[str] = "__openhcs_raw_runtime_signature__" processing_prepare: ClassVar[str] = "__openhcs_prepare__" runtime_adapter: ClassVar[str] = "__runtime_adapter__" runtime_image_execution_mode: ClassVar[str] = ( diff --git a/openhcs/core/function_patterns.py b/openhcs/core/function_patterns.py index b79b4fc695..f41e9ad424 100644 --- a/openhcs/core/function_patterns.py +++ b/openhcs/core/function_patterns.py @@ -1265,7 +1265,7 @@ def normalize( func, kwargs = _split_function_item(item) if RUNTIME_CALLABLE_KWARG_POLICY.item_is_disabled(kwargs): continue - contract = CallableContract.from_callable(func) + contract = CallableContract.from_prepared_callable(func) position = len(normalized_items) normalized_items.append( NormalizedFunctionItem( diff --git a/openhcs/core/pipeline/function_contracts.py b/openhcs/core/pipeline/function_contracts.py index b75413a3a9..413f266de8 100644 --- a/openhcs/core/pipeline/function_contracts.py +++ b/openhcs/core/pipeline/function_contracts.py @@ -3,7 +3,6 @@ from collections.abc import Sequence from dataclasses import replace from enum import Enum -from functools import lru_cache import inspect from types import UnionType from typing import ( @@ -43,11 +42,19 @@ F = TypeVar("F", bound=Callable) -@lru_cache(maxsize=256) def resolved_callable_type_hints(func: Callable) -> dict[str, Any]: """Return the callable's resolved type contract or propagate its error.""" - return get_type_hints(func) + signature = CallableMetadata.prepared_callable_signature(func) + if signature is None: + return get_type_hints(func) + from typing import _strip_annotations + + return { + **{name: _strip_annotations(parameter.annotation) for name, parameter in signature.parameters.items() + if parameter.annotation is not inspect.Parameter.empty}, + **({} if signature.return_annotation is inspect.Signature.empty else {"return": _strip_annotations(signature.return_annotation)}), + } def annotation_accepts_runtime_type(annotation: object, value_type: type[Any]) -> bool: @@ -101,7 +108,7 @@ def resolved_callable_parameter( ) -> inspect.Parameter: """Return one callable parameter with its runtime type hint resolved.""" - signature = inspect.signature(func) + signature = CallableMetadata.callable_signature(func) parameter = signature.parameters.get(parameter_name) if parameter is None: raise ValueError( @@ -429,7 +436,7 @@ def special_input_parameters_from_callable( """Return special-input parameters in their canonical signature order.""" declared_names = special_input_names_from_callable(func) - signature = inspect.signature(func) + signature = CallableMetadata.callable_signature(func) missing = tuple(name for name in declared_names if name not in signature.parameters) if missing: raise ValueError( diff --git a/openhcs/core/runtime_batch_contracts.py b/openhcs/core/runtime_batch_contracts.py index 6933013d51..cf48ff9032 100644 --- a/openhcs/core/runtime_batch_contracts.py +++ b/openhcs/core/runtime_batch_contracts.py @@ -6,14 +6,13 @@ from collections.abc import Callable, Hashable, Mapping from dataclasses import dataclass, replace from enum import Enum -from functools import lru_cache import inspect from types import MappingProxyType from typing import ClassVar, Generic, TypeVar from metaclass_registry import AutoRegisterMeta -from openhcs.core.callable_contract import KeywordRuntimeParameter +from openhcs.core.callable_contract import CallableMetadata, KeywordRuntimeParameter from openhcs.core.aligned_image_payload import ( AlignedImageStack, ImagePayloadExecutionMode, @@ -48,11 +47,12 @@ class RuntimePlaneAxisValueProjectionParameter(KeywordRuntimeParameter): parameter_default = None -@lru_cache(maxsize=1024) -def runtime_callable_defaults(func: Callable[..., object]) -> Mapping[str, object]: +def runtime_callable_defaults( + func: Callable[..., object], *, signature: inspect.Signature | None = None, +) -> Mapping[str, object]: """Return callable defaults visible to runtime batch executors.""" try: - callable_signature = inspect.signature(func) + callable_signature = CallableMetadata.callable_signature(func) if signature is None else signature except (TypeError, ValueError) as exc: raise TypeError( f"Runtime batch function {func!r} must expose an inspectable signature." @@ -190,6 +190,7 @@ class RuntimePure2DSliceBatchRequest( ], RuntimeSliceResultT, ] + signature: inspect.Signature | None = None def __post_init__(self) -> None: object.__setattr__( @@ -197,7 +198,7 @@ def __post_init__(self) -> None: "kwargs", MappingProxyType( { - **runtime_callable_defaults(self.func), + **runtime_callable_defaults(self.func, signature=self.signature), **dict(self.kwargs), } ), diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index 948d937ec2..21f0ea6762 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -43,7 +43,6 @@ ) from openhcs.core.callable_contract import ( CallableRuntimeCacheKey, - prepare_processing_callable, ) from openhcs.core.component_group_scope import ( ComponentGroupScope, @@ -245,9 +244,6 @@ class FunctionInvocationCallableResolver: def prepare(cls, invocation: CompiledFunctionInvocation) -> None: """Resolve and cache one invocation callable before timed execution.""" cls.resolve(invocation) - prepare_processing_callable( - invocation.contract.resolve_canonical_raw_callable() - ) @classmethod def resolve(cls, invocation: CompiledFunctionInvocation) -> Callable: diff --git a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py index 7ce0d2275a..76ab77ecca 100644 --- a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py +++ b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py @@ -57,6 +57,7 @@ RuntimeCallablePolicy, RuntimeCallableView, RuntimeInvocationKwargPolicy, + contextualize_main_image_output, ) _CELLPROFILER_RUNTIME_CALLABLE_POLICY = RuntimeCallablePolicy( @@ -75,6 +76,21 @@ def __init__( ) -> None: self.plane_projection = plane_projection + def invoke_raw( + self, + callable_contract: CallableContract, + func: Callable[..., RuntimeFunctionOutput], + image: RuntimeCallableArgument, + kwargs: RuntimeCallableKwargs, + ) -> RuntimeCallableArgument: + """Project the canonical ABI after the input domain has been selected.""" + return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + func, + (callable_contract.raw_main_flow_call_argument(image),), + kwargs, + signature=callable_contract.raw_runtime_signature, + ).call() + def execute( self, callable_contract: CallableContract, @@ -135,7 +151,7 @@ def execute( runtime_kwargs.update( processing_contract.declaration.consume_semantic_controls( runtime_kwargs, - func=func, + parameters=callable_contract.canonical_signature.parameters, ) ) result = processing_contract.execute( @@ -211,6 +227,7 @@ def execute_pure_2d_slice_batch( slices_2d=slices_2d, kwargs=kwargs, execute_slice=execute_slice, + signature=callable_contract.raw_runtime_signature, ) if slice_request.slice_count <= 0: return [], 0.0 @@ -261,11 +278,12 @@ def execute_pure_3d( ), ) call_started_at = time.perf_counter() - result = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + result = self.invoke_raw( + callable_contract, func, - (projected_image,), + projected_image, projected_kwargs, - ).call() + ) CellProfilerRuntimeProfileLogger.log_module_profile( "cp_full_stack_raw_call", time.perf_counter() - call_started_at, @@ -339,16 +357,17 @@ def execute_aligned_stack_slice( slice_index: int, slice_count: int, ) -> RuntimeCallableArgument: - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + return self.invoke_raw( + callable_contract, slice_func, - (slice_payload,), + slice_payload, aligned_image_stack_kwargs( slice_kwargs, slice_index, slice_count, reference_payload=slice_payload, ), - ).call() + ) slice_results, _slice_execute_seconds = self.execute_pure_2d_slice_batch( callable_contract, @@ -457,11 +476,12 @@ def execute_pure_2d( ) image_data = image_payload_data(image) if not isinstance(image_data, np.ndarray): - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + return self.invoke_raw( + callable_contract, func, - (image,), + image, kwargs, - ).call() + ) prepare_started_at = time.perf_counter() memory_type = detect_memory_type(image_data) @@ -484,11 +504,12 @@ def execute_pure_2d( "declared plane projection. Kwargs cannot create image-axis " "execution semantics." ) - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + return self.invoke_raw( + callable_contract, func, - (image,), + image, kwargs, - ).call() + ) declared_plane_axis = image_payload_metadata(image).plane_axis if declared_plane_axis is not self.plane_projection.axis: raise RuntimeSliceProjectionDeclarationError( @@ -588,12 +609,12 @@ def execute_volumetric_to_slice( callable_contract: CallableContract, **kwargs: RuntimeCallableArgument, ) -> RuntimeCallableArgument: - del callable_contract - result_2d = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + result_2d = self.invoke_raw( + callable_contract, func, - (image,), + image, kwargs, - ).call() + ) result_data = image_payload_data(result_2d) result_mask = image_payload_mask(result_2d) result_metadata = image_payload_metadata(result_2d) @@ -621,11 +642,13 @@ def execute_pure_2d_slice( sliced_kwargs = dict(sliced_kwargs) slice_index_name = SliceIndexRuntimeParameter.require_parameter_name() sliced_kwargs[slice_index_name] = slice_index - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( + result = self.invoke_raw( + callable_contract, func, - (slice_2d,), + slice_2d, sliced_kwargs, - ).call() + ) + return contextualize_main_image_output(slice_2d, result) def slice_pure_2d_kwargs( self, diff --git a/openhcs/processing/backends/cellprofiler/classification.py b/openhcs/processing/backends/cellprofiler/classification.py index 4783e7e795..e7e22bd10a 100644 --- a/openhcs/processing/backends/cellprofiler/classification.py +++ b/openhcs/processing/backends/cellprofiler/classification.py @@ -2028,7 +2028,7 @@ def object_classification_backend( SliceIndexRuntimeParameter, ) def classify_objects_single_measurement( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, measurement_feature: str = "", measurement_values: np.ndarray | None = None, diff --git a/openhcs/processing/backends/cellprofiler/colocalization.py b/openhcs/processing/backends/cellprofiler/colocalization.py index 12e4d220ff..75b4ae097e 100644 --- a/openhcs/processing/backends/cellprofiler/colocalization.py +++ b/openhcs/processing/backends/cellprofiler/colocalization.py @@ -2498,7 +2498,7 @@ def _colocalization_unit_interval_scale( @numpy(contract=ProcessingContract.FLEXIBLE) @runtime_bound_parameters(_ColocalizationThresholdMaskOutputsRuntimeParameter) def measure_colocalization( - image: np.ndarray, + image: RuntimeArrayData, channel_1: int = 0, channel_2: int = 1, threshold_percent: float = 15.0, @@ -2766,7 +2766,7 @@ def _measure_colocalization_objects_core( _ColocalizationCostesThresholdBatchRuntimeParameter, ) def measure_colocalization_objects( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, measurement_scope: CellProfilerMeasurementTargetScope = CellProfilerMeasurementTargetScope.OBJECT, channel_1: int = 0, @@ -2857,7 +2857,7 @@ def measure_colocalization_objects( def _colocalization_threshold_mask_canonical_output( - image: np.ndarray, + image: RuntimeArrayData, *, threshold_mask_groups: tuple[ColocalizationThresholdMaskGroup, ...], threshold_mask_outputs: tuple[ColocalizationThresholdMaskRuntimeOutput, ...], @@ -2887,7 +2887,7 @@ def _colocalization_threshold_mask_canonical_output( def _colocalization_threshold_mask( - image: np.ndarray, + image: RuntimeArrayData, request: ColocalizationThresholdMaskRuntimeOutput, ) -> RuntimeArrayData: """Apply CellProfiler's whole-image or per-object percentage threshold.""" diff --git a/openhcs/processing/backends/cellprofiler/color.py b/openhcs/processing/backends/cellprofiler/color.py index b7e5ad86ca..3bd92535dd 100644 --- a/openhcs/processing/backends/cellprofiler/color.py +++ b/openhcs/processing/backends/cellprofiler/color.py @@ -1494,7 +1494,7 @@ def run(self, request: GrayToColorRequest) -> np.ndarray: @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def gray_to_color( - image: np.ndarray, + image: RuntimeArrayData, color_scheme: GrayToColorModule.Scheme = GrayToColorModule.Scheme.RGB, rescale_intensity: bool = True, red_channel: int = -1, @@ -1593,7 +1593,7 @@ def gray_to_color( ) @numpy(contract=ProcessingContract.FLEXIBLE) def color_to_gray( - image: np.ndarray, + image: RuntimeArrayData, mode: ColorToGrayMode = ColorToGrayMode.SPLIT, image_type: ImageChannelType = ImageChannelType.RGB, channel_indices: tuple[int, ...] = ColorToGrayModule.default_channel_indices, @@ -1634,7 +1634,7 @@ def color_to_gray( def _invert_for_printing_channels( - image: np.ndarray, + image: RuntimeArrayData, *, input_mode: InvertInputMode, use_red_input: bool, @@ -1676,7 +1676,7 @@ def _invert_for_printing_channels( def _invert_for_printing_result( - image: np.ndarray, + image: RuntimeArrayData, *, input_mode: InvertInputMode, use_red_input: bool, @@ -1758,7 +1758,7 @@ def _invert_for_printing_result( @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def invert_for_printing( - image: np.ndarray, + image: RuntimeArrayData, input_mode: InvertInputMode = InvertInputMode.COLOR, use_red_input: bool = True, use_green_input: bool = True, @@ -1794,7 +1794,7 @@ def invert_for_printing( @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def invert_for_printing_grayscale( - image: np.ndarray, + image: RuntimeArrayData, input_mode: InvertInputMode = InvertInputMode.COLOR, use_red_input: bool = True, use_green_input: bool = True, @@ -1837,7 +1837,7 @@ def invert_for_printing_grayscale( @composed_image_payload @numpy(contract=ProcessingContract.PURE_3D) def invert_for_printing_without_output( - image: np.ndarray, + image: RuntimeArrayData, input_mode: InvertInputMode = InvertInputMode.COLOR, use_red_input: bool = True, use_green_input: bool = True, @@ -1876,7 +1876,7 @@ def invert_for_printing_without_output( def combine_color_to_gray( - image: np.ndarray, + image: RuntimeArrayData, channel_indices: tuple[int, ...], contributions: tuple[float, ...], ) -> np.ndarray: @@ -1892,7 +1892,7 @@ def combine_color_to_gray( return restore_color_to_gray_shape(image, result) -def color_to_gray_combine_output_metadata(image: np.ndarray): +def color_to_gray_combine_output_metadata(image: RuntimeArrayData): """Return metadata for a color-to-grayscale semantic collapse.""" return ( image_payload_metadata(image) @@ -1902,7 +1902,7 @@ def color_to_gray_combine_output_metadata(image: np.ndarray): def split_color_to_gray( - image: np.ndarray, image_type: ImageChannelType, channel_indices: tuple[int, ...] + image: RuntimeArrayData, image_type: ImageChannelType, channel_indices: tuple[int, ...] ) -> tuple[np.ndarray, ...]: color_stack = nhwc_color_stack(image).astype(np.float32) source_stack = ( @@ -1928,7 +1928,7 @@ def color_to_gray_channel(color_stack: np.ndarray, channel_index: int) -> np.nda return color_stack[..., channel_index] -def nhwc_color_stack(image: np.ndarray) -> np.ndarray: +def nhwc_color_stack(image: RuntimeArrayData) -> np.ndarray: """Return NHWC pixels from explicitly declared image layout metadata.""" image_data = np.asarray(image_payload_data(image)) metadata = image_payload_metadata(image) @@ -1959,7 +1959,7 @@ def nhwc_color_stack(image: np.ndarray) -> np.ndarray: return channel_last -def restore_color_to_gray_shape(original: np.ndarray, stack: np.ndarray) -> np.ndarray: +def restore_color_to_gray_shape(original: RuntimeArrayData, stack: np.ndarray) -> np.ndarray: metadata = image_payload_metadata(original) if metadata.is_declared_source_channel_plane(original): if stack.shape[0] != 1: @@ -2017,7 +2017,7 @@ def rgb_to_hsv_stack(rgb_stack: np.ndarray) -> np.ndarray: @numpy(contract=ProcessingContract.FLEXIBLE) def unmix_colors( - image: np.ndarray, + image: RuntimeArrayData, stain_names: Sequence[StainType] = (), custom_absorbances: Sequence[Sequence[float] | None] = (), stain1: StainType = StainType.HEMATOXYLIN, diff --git a/openhcs/processing/backends/cellprofiler/display_modules.py b/openhcs/processing/backends/cellprofiler/display_modules.py index 28c40ce4f4..d5566f0a94 100644 --- a/openhcs/processing/backends/cellprofiler/display_modules.py +++ b/openhcs/processing/backends/cellprofiler/display_modules.py @@ -29,6 +29,7 @@ special_inputs, ) from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_metadata, @@ -492,7 +493,7 @@ def render_slice(self) -> np.ndarray: @special_inputs("labels") @runtime_bound_parameters(_DisplayMeasurementsRuntimeParameter) def display_data_on_image( - image: np.ndarray, + image: RuntimeArrayData, labels: Optional[ObjectLabelValue] = None, measurements: Optional[np.ndarray] = None, measurement_feature: Optional[str] = None, diff --git a/openhcs/processing/backends/cellprofiler/edge.py b/openhcs/processing/backends/cellprofiler/edge.py index 30bc44280d..5e01b30a19 100644 --- a/openhcs/processing/backends/cellprofiler/edge.py +++ b/openhcs/processing/backends/cellprofiler/edge.py @@ -50,6 +50,7 @@ class EnhanceEdgesModule(CellProfilerModule): from numba import njit from openhcs.core.memory.decorators import numpy as numpy_decorator from openhcs.core.public_api import public_names_from_objects +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_mask, @@ -508,7 +509,7 @@ def _native_kirsch(image: np.ndarray) -> np.ndarray: @numpy_decorator(contract=ProcessingContract.PURE_2D) def enhance_edges( - image: np.ndarray, + image: RuntimeArrayData, method: EdgeMethod = EdgeMethod.SOBEL, direction: EdgeDirection = EdgeDirection.ALL, edge_backend_provider: BackendProviderInput = DEFAULT_CELLPROFILER_BACKEND_SELECTION, diff --git a/openhcs/processing/backends/cellprofiler/feature_enhancement.py b/openhcs/processing/backends/cellprofiler/feature_enhancement.py index b8f6bd7bda..a2b813799b 100644 --- a/openhcs/processing/backends/cellprofiler/feature_enhancement.py +++ b/openhcs/processing/backends/cellprofiler/feature_enhancement.py @@ -13,6 +13,7 @@ from openhcs.core.callable_contract import processing_prepare from openhcs.core.memory.decorators import numpy from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_mask, @@ -65,7 +66,7 @@ class NeuriteMethod(Enum): @numpy(contract=ProcessingContract.PURE_2D) def enhance_or_suppress_features( - image: np.ndarray, + image: RuntimeArrayData, method: OperationMethod = OperationMethod.ENHANCE, enhance_method: EnhanceMethod = EnhanceMethod.SPECKLES, radius: float = 10.0, diff --git a/openhcs/processing/backends/cellprofiler/gaussian_filter.py b/openhcs/processing/backends/cellprofiler/gaussian_filter.py index 187d4d972d..7018d62595 100644 --- a/openhcs/processing/backends/cellprofiler/gaussian_filter.py +++ b/openhcs/processing/backends/cellprofiler/gaussian_filter.py @@ -7,6 +7,7 @@ from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode from openhcs.core.callable_contract import runtime_image_execution_mode +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_metadata, @@ -41,7 +42,7 @@ class GaussianFilterModule(CellProfilerModule): @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy(contract=ProcessingContract.FLEXIBLE) -def gaussian_filter(image: np.ndarray, sigma: float = 1.0) -> np.ndarray: +def gaussian_filter(image: RuntimeArrayData, sigma: float = 1.0) -> np.ndarray: """ Apply CellProfiler-compatible Gaussian smoothing to an image. diff --git a/openhcs/processing/backends/cellprofiler/illumination.py b/openhcs/processing/backends/cellprofiler/illumination.py index db2ba6b1f3..d86e17def8 100644 --- a/openhcs/processing/backends/cellprofiler/illumination.py +++ b/openhcs/processing/backends/cellprofiler/illumination.py @@ -920,7 +920,7 @@ def image_payload( @numpy(contract=ProcessingContract.FLEXIBLE) def correct_illumination_calculate( - image: np.ndarray, + image: RuntimeArrayData, intensity_choice: IntensityChoice = IntensityChoice.REGULAR, dilate_objects: bool = False, object_dilation_radius: int = 1, diff --git a/openhcs/processing/backends/cellprofiler/image_geometry.py b/openhcs/processing/backends/cellprofiler/image_geometry.py index 61bdddcca5..37b24b3ce3 100644 --- a/openhcs/processing/backends/cellprofiler/image_geometry.py +++ b/openhcs/processing/backends/cellprofiler/image_geometry.py @@ -1093,7 +1093,7 @@ def align_label_plane_to_shape( @numpy(contract=ProcessingContract.FLEXIBLE) @special_inputs("mask") def mask_image( - image: np.ndarray, + image: RuntimeArrayData, mask: np.ndarray | ObjectLabelValue, mask_source: MaskSource = MaskSource.IMAGE, invert_mask: bool = False, @@ -1130,7 +1130,7 @@ def mask_image( def masked_image_plane( - image: np.ndarray, binary_mask: np.ndarray, *, invert_mask: bool + image: RuntimeArrayData, binary_mask: np.ndarray, *, invert_mask: bool ) -> tuple[np.ndarray, np.ndarray]: image_data = image_payload_data(image) if invert_mask: @@ -1159,7 +1159,7 @@ def masked_image_plane( @numpy(contract=ProcessingContract.PURE_2D) def mask_image_with_binary( - image: np.ndarray, invert_mask: bool = False + image: RuntimeArrayData, invert_mask: bool = False ) -> RuntimeArrayData: """Return a binary mask plane, optionally inverted.""" binary_mask = image_payload_data(image) > 0.5 @@ -1220,7 +1220,7 @@ def tile_output_shape( @numpy(contract=ProcessingContract.FLEXIBLE) def tile( - image: np.ndarray, + image: RuntimeArrayData, rows: int = 8, columns: int = 12, place_first: PlaceFirst = PlaceFirst.TOP_LEFT, @@ -1322,7 +1322,7 @@ def resize_volumetric( @numpy(contract=ProcessingContract.PURE_2D) def flip_and_rotate( - image: np.ndarray, + image: RuntimeArrayData, flip_method: FlipMethod = FlipMethod.NONE, rotate_method: RotateMethod = RotateMethod.NONE, rotation_angle: float = 0.0, diff --git a/openhcs/processing/backends/cellprofiler/intensity.py b/openhcs/processing/backends/cellprofiler/intensity.py index cbb6dbcde0..aae7b8583d 100644 --- a/openhcs/processing/backends/cellprofiler/intensity.py +++ b/openhcs/processing/backends/cellprofiler/intensity.py @@ -302,6 +302,7 @@ def active_artifact_bindings( from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImageMetadataPayload, MaskedImagePayload, @@ -1856,7 +1857,7 @@ def run(self, context: RescaleIntensityContext) -> np.ndarray: @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.PURE_2D) def rescale_intensity( - image: np.ndarray, + image: RuntimeArrayData, rescale_method: RescaleMethod = RescaleMethod.STRETCH, automatic_low: AutomaticLow = AutomaticLow.EACH_IMAGE, automatic_high: AutomaticHigh = AutomaticHigh.EACH_IMAGE, diff --git a/openhcs/processing/backends/cellprofiler/intensity_distribution.py b/openhcs/processing/backends/cellprofiler/intensity_distribution.py index c9c89ba06d..75d78d4341 100644 --- a/openhcs/processing/backends/cellprofiler/intensity_distribution.py +++ b/openhcs/processing/backends/cellprofiler/intensity_distribution.py @@ -2461,7 +2461,7 @@ def _radial_distribution_arrays_from_bin_totals_numba( _IntensityDistributionHeatmapOutputsRuntimeParameter, ) def measure_object_intensity_distribution( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, bin_count: int = 4, wants_scaled: bool = True, @@ -2562,7 +2562,7 @@ def measure_object_intensity_distribution( def _intensity_distribution_heatmap( - image: np.ndarray, + image: RuntimeArrayData, request: IntensityDistributionHeatmapRuntimeOutput, *, radial_backend: "RadialDistributionBackendStrategy", diff --git a/openhcs/processing/backends/cellprofiler/median_filter.py b/openhcs/processing/backends/cellprofiler/median_filter.py index ca24bd2c00..ec19edfcc9 100644 --- a/openhcs/processing/backends/cellprofiler/median_filter.py +++ b/openhcs/processing/backends/cellprofiler/median_filter.py @@ -19,6 +19,7 @@ RuntimePure2DSliceBatchRequest, pure_2d_batch_executor, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, with_image_payload_data, @@ -371,7 +372,7 @@ def median_filter_backend( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy(contract=ProcessingContract.FLEXIBLE) def medianfilter( - image: np.ndarray, + image: RuntimeArrayData, window_size: int = 3, mode: ScipyBoundaryMode = ScipyBoundaryMode.CONSTANT, ) -> np.ndarray: diff --git a/openhcs/processing/backends/cellprofiler/morphology.py b/openhcs/processing/backends/cellprofiler/morphology.py index ae58161498..8247ca9b44 100644 --- a/openhcs/processing/backends/cellprofiler/morphology.py +++ b/openhcs/processing/backends/cellprofiler/morphology.py @@ -690,6 +690,7 @@ def artifact_output_relations( object_label_identity_lineage_payload, object_label_parent_child_payload, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, @@ -1491,7 +1492,7 @@ def _morph_image_pixels( def _morph_image_payload( - image: np.ndarray, + image: RuntimeArrayData, structuring_element: StructuringElement, size: int, operation: Callable[[np.ndarray, np.ndarray], np.ndarray], @@ -1508,7 +1509,7 @@ def _morph_image_payload( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.FLEXIBLE) def closing( - image: np.ndarray, + image: RuntimeArrayData, structuring_element: StructuringElementInput = StructuringElement.DISK, size: StructuringElementSize = 3, morphology_backend_provider: BackendProviderInput = CellProfilerBackendProvider.NATIVE, @@ -1530,7 +1531,7 @@ def closing( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.FLEXIBLE) def opening( - image: np.ndarray, + image: RuntimeArrayData, structuring_element: StructuringElementInput = StructuringElement.DISK, size: StructuringElementSize = 3, morphology_backend_provider: BackendProviderInput = CellProfilerBackendProvider.NATIVE, diff --git a/openhcs/processing/backends/cellprofiler/outlines.py b/openhcs/processing/backends/cellprofiler/outlines.py index cdaeb5cefd..d0e4791fe7 100644 --- a/openhcs/processing/backends/cellprofiler/outlines.py +++ b/openhcs/processing/backends/cellprofiler/outlines.py @@ -878,7 +878,7 @@ def render_single_plane(self, image_sources: tuple[np.ndarray, ...]) -> np.ndarr @object_label_input_execution_mode(ObjectLabelInputExecutionMode.MATCH_IMAGE_STACK) @special_inputs("object_labels") def overlay_outlines( - image: np.ndarray, + image: RuntimeArrayData, *, blank_image: bool = False, display_mode: OutlineDisplayMode = OutlineDisplayMode.COLOR, @@ -922,7 +922,7 @@ def overlay_outlines( @object_label_input_execution_mode(ObjectLabelInputExecutionMode.MATCH_IMAGE_STACK) @special_inputs("labels") def overlay_objects( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, opacity: float = 0.3, max_label: int | None = None, @@ -1000,7 +1000,7 @@ def overlay_objects( def _overlay_objects_array( - image: np.ndarray, + image: RuntimeArrayData, labels: np.ndarray, *, opacity: float, @@ -1181,7 +1181,7 @@ def _draw_object_labels( def _draw_outline_image( output: np.ndarray, - outline_image: np.ndarray, + outline_image: RuntimeArrayData, color: tuple[float, float, float], *, outline_intensity: float, @@ -1200,7 +1200,7 @@ def _draw_outline_image( return output -def _outline_image_mask(outline_image: np.ndarray) -> np.ndarray: +def _outline_image_mask(outline_image: RuntimeArrayData) -> np.ndarray: mask = np.asarray(image_payload_data(outline_image)) > 0 channel_axis = image_payload_metadata(outline_image).normalized_source_channel_axis( outline_image diff --git a/openhcs/processing/backends/cellprofiler/primary_objects.py b/openhcs/processing/backends/cellprofiler/primary_objects.py index 70c083129a..9f51d8e0ba 100644 --- a/openhcs/processing/backends/cellprofiler/primary_objects.py +++ b/openhcs/processing/backends/cellprofiler/primary_objects.py @@ -11,6 +11,7 @@ from enum import Enum from openhcs.core.memory import numpy from openhcs.core.public_api import public_names_from_objects +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, @@ -123,7 +124,7 @@ class ExcessObjectHandling(CellProfilerEnumAttributeMixin, Enum): @numpy(contract=ProcessingContract.PURE_2D) def identify_primary_objects( - image: np.ndarray, + image: RuntimeArrayData, min_diameter: int = 10, max_diameter: int = 40, exclude_size: bool = True, diff --git a/openhcs/processing/backends/cellprofiler/secondary.py b/openhcs/processing/backends/cellprofiler/secondary.py index 973a360374..d31dbc0f8c 100644 --- a/openhcs/processing/backends/cellprofiler/secondary.py +++ b/openhcs/processing/backends/cellprofiler/secondary.py @@ -38,6 +38,7 @@ EnumKeyedStrategyMixin, RegisteredLeafClassSpec, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, image_payload_mask, @@ -1147,7 +1148,7 @@ def _replacement_primary_output_from_relationship( def _execute_identify_secondary_objects( - image: np.ndarray, + image: RuntimeArrayData, primary_labels: ObjectLabelValue, method: SecondaryMethod = SecondaryMethod.PROPAGATION, threshold_scope: CellProfilerThresholdScope = CellProfilerThresholdScope.GLOBAL, @@ -1337,7 +1338,7 @@ def _execute_identify_secondary_objects( @numpy(contract=ProcessingContract.PURE_2D) @special_inputs("primary_labels") def identify_secondary_objects( - image: np.ndarray, + image: RuntimeArrayData, primary_labels: ObjectLabelValue, method: SecondaryMethod = SecondaryMethod.PROPAGATION, threshold_scope: CellProfilerThresholdScope = CellProfilerThresholdScope.GLOBAL, @@ -1407,7 +1408,7 @@ def identify_secondary_objects( @numpy(contract=ProcessingContract.PURE_2D) @special_inputs("primary_labels") def identify_secondary_objects_with_replacement_primary( - image: np.ndarray, + image: RuntimeArrayData, primary_labels: ObjectLabelValue, method: SecondaryMethod = SecondaryMethod.PROPAGATION, threshold_scope: CellProfilerThresholdScope = CellProfilerThresholdScope.GLOBAL, diff --git a/openhcs/processing/backends/cellprofiler/skeleton.py b/openhcs/processing/backends/cellprofiler/skeleton.py index 947ca74c01..ba8c710ced 100644 --- a/openhcs/processing/backends/cellprofiler/skeleton.py +++ b/openhcs/processing/backends/cellprofiler/skeleton.py @@ -359,7 +359,7 @@ def measure_object_skeleton( @numpy_backend(contract=ProcessingContract.PURE_2D) @special_inputs("seed_labels") def measure_object_skeleton_with_branchpoint_image( - image: np.ndarray, + image: RuntimeArrayData, seed_labels: SeedObjectLabelsInput, fill_small_holes: bool = True, maximum_hole_size: int = 10, diff --git a/openhcs/processing/backends/cellprofiler/smoothing.py b/openhcs/processing/backends/cellprofiler/smoothing.py index 0b803b2294..6ac97257a9 100644 --- a/openhcs/processing/backends/cellprofiler/smoothing.py +++ b/openhcs/processing/backends/cellprofiler/smoothing.py @@ -5,6 +5,7 @@ from typing import Any from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode from openhcs.core.callable_contract import runtime_image_execution_mode +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( image_payload_data, with_image_payload_data, @@ -790,7 +791,7 @@ def _fit_polynomial( def smooth_image( - image: np.ndarray, + image: RuntimeArrayData, smoothing_method: SmoothingMethod = SmoothingMethod.GAUSSIAN_FILTER, auto_object_size: bool = True, object_size: float = 16.0, @@ -839,7 +840,7 @@ def smooth_image( @numpy_decorator(contract=ProcessingContract.PURE_2D) def smooth( - image: np.ndarray, + image: RuntimeArrayData, smoothing_method: SmoothingMethod = SmoothingMethod.GAUSSIAN_FILTER, auto_object_size: bool = True, object_size: float = 16.0, @@ -862,7 +863,7 @@ def smooth( @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy_decorator(contract=ProcessingContract.FLEXIBLE) def reducenoise( - image: np.ndarray, + image: RuntimeArrayData, patch_size: int = 5, patch_distance: int = 6, cutoff_distance: float = 0.1, diff --git a/openhcs/processing/backends/cellprofiler/worms.py b/openhcs/processing/backends/cellprofiler/worms.py index 240070daef..74359cf712 100644 --- a/openhcs/processing/backends/cellprofiler/worms.py +++ b/openhcs/processing/backends/cellprofiler/worms.py @@ -2112,7 +2112,7 @@ def untangle_worms_both( @special_inputs("worm_labels") @runtime_bound_parameters(_StraightenWormControlPointsRuntimeParameter) def straighten_worms( - image: np.ndarray, + image: RuntimeArrayData, worm_labels: ObjectLabelValue, control_points: np.ndarray | None = None, worm_width: int = 20, diff --git a/openhcs/processing/backends/lib_registry/registry_service.py b/openhcs/processing/backends/lib_registry/registry_service.py index 5f1c3d226a..36459249c1 100644 --- a/openhcs/processing/backends/lib_registry/registry_service.py +++ b/openhcs/processing/backends/lib_registry/registry_service.py @@ -21,7 +21,7 @@ from zmqruntime import OperationCancellation from zmqruntime.client import endpoint_process -from openhcs.core.callable_contract import CallableProjection +from openhcs.core.callable_contract import CallableContract, CallableProjection from openhcs.core.function_reference import ResolvedRegistryFunction from openhcs.core.processing_preparation import ( CallablePreparation, @@ -145,11 +145,11 @@ def prepare_in_current_process( cls._metadata_cache = cls._metadata_from_instances( cls._available_registry_instances() ) - callables = tuple( + callables = tuple(dict.fromkeys( target for metadata in cls._metadata_cache.values() for target in CallableProjection.from_callable(metadata.func).prepare_targets() - ) + )) emit_status("Preparing registered kernel caches") PreparationCacheBatch.from_callables(callables).populate_child_caches( max_workers=os.cpu_count() or 1, @@ -161,6 +161,9 @@ def prepare_in_current_process( emit_status( f"Prepared callable {preparation.projection.module_name}.{preparation.projection.name}" ) + emit_status("Resolving prepared callable signatures") + for func in callables: + CallableContract.warm_canonical_signature(func) emit_status(f"Registered kernels ready ({len(callables)} callables)") emit_status(f"Function catalog ready ({len(cls._metadata_cache)} callables)") return cls._metadata_cache diff --git a/openhcs/processing/backends/lib_registry/unified_registry.py b/openhcs/processing/backends/lib_registry/unified_registry.py index 4e60c05d3d..f6097a5e74 100644 --- a/openhcs/processing/backends/lib_registry/unified_registry.py +++ b/openhcs/processing/backends/lib_registry/unified_registry.py @@ -207,6 +207,7 @@ def accepted_kwargs( self, func: Callable[..., Any], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> dict[str, Any]: """Return kwargs accepted by ``func`` under this policy.""" @@ -220,8 +221,9 @@ def accepted_kwargs( self, func: Callable[..., Any], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> dict[str, Any]: - del func + del func, signature return dict(kwargs) @@ -234,8 +236,11 @@ def accepted_kwargs( self, func: Callable[..., Any], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> dict[str, Any]: - parameters = _runtime_callable_parameters(func) + from openhcs.core.callable_contract import CallableMetadata + + parameters = (CallableMetadata.callable_signature(func) if signature is None else signature).parameters if any( parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() @@ -255,6 +260,7 @@ class RuntimeCallableInvocation: kwarg_policy: RuntimeInvocationKwargPolicy = ( RuntimeInvocationKwargPolicy.PASS_THROUGH ) + signature: inspect.Signature | None = None def call(self) -> Any: target = RuntimeCallableViewStrategy.for_view(self.callable_view).resolve( @@ -264,7 +270,7 @@ def call(self) -> Any: *self.args, **RuntimeInvocationKwargPolicyStrategy.for_policy( self.kwarg_policy - ).accepted_kwargs(target, self.kwargs), + ).accepted_kwargs(target, self.kwargs, signature=self.signature), ) @@ -282,6 +288,7 @@ def invocation( func: Callable[..., Any], args: tuple[Any, ...], kwargs: Mapping[str, Any], + *, signature: inspect.Signature | None = None, ) -> RuntimeCallableInvocation: return RuntimeCallableInvocation( func=func, @@ -289,16 +296,10 @@ def invocation( kwargs=kwargs, callable_view=self.callable_view, kwarg_policy=self.kwarg_policy, + signature=signature, ) -@lru_cache(maxsize=256) -def _runtime_callable_parameters( - func: Callable[..., Any], -) -> Mapping[str, inspect.Parameter]: - return inspect.signature(func).parameters - - def _registry_runtime_parameter_exclusions( signature: inspect.Signature, parameter_names: tuple[str, ...], @@ -1012,7 +1013,7 @@ def main_flow_call_argument( self, callable_contract: "CallableContract", source_payload: Any, ) -> Any: """Expose the raw ABI when this contract needs no earlier plane slicing.""" - return callable_contract.raw_main_flow_call_argument(source_payload) + return callable_contract.runtime_main_flow_call_argument(source_payload) def injected_semantic_control_parameter_names(self) -> frozenset[str]: """Semantic controls that this contract may inject into public callables.""" @@ -1027,10 +1028,14 @@ def consume_semantic_controls( kwargs: MutableMapping[str, Any], *, func: Callable[..., Any] | None = None, + parameters: Mapping[str, inspect.Parameter] | None = None, ) -> dict[str, Any]: """Consume and return semantic selectors owned by this declaration.""" - parameters = {} if func is None else inspect.signature(func).parameters + if parameters is None: + from openhcs.core.callable_contract import CallableMetadata + + parameters = {} if func is None else CallableMetadata.callable_signature(func).parameters values: dict[str, Any] = {} for parameter_type in self.runtime_parameter_types(): if not parameter_type.is_semantic_control: @@ -1739,7 +1744,7 @@ def execute_pure_3d(self, func, image, *args, **kwargs): RuntimeCallablePolicy() .invocation( func, - (callable_contract.raw_main_flow_call_argument(image), *args), + (callable_contract.runtime_main_flow_call_argument(image), *args), kwargs, ) .call() @@ -1773,7 +1778,7 @@ def execute_pure_2d(self, func, image, *args, **kwargs): result = ( RuntimeCallablePolicy().invocation( func, - (callable_contract.raw_main_flow_call_argument(image), *args), + (callable_contract.runtime_main_flow_call_argument(image), *args), kwargs, ).call() ) @@ -1837,7 +1842,7 @@ def execute_slice( RuntimeCallablePolicy() .invocation( slice_func, - (callable_contract.raw_main_flow_call_argument(slice_2d), *args), + (callable_contract.runtime_main_flow_call_argument(slice_2d), *args), projected_kwargs, ) .call() diff --git a/tests/unit/test_callable_abi_preparation.py b/tests/unit/test_callable_abi_preparation.py new file mode 100644 index 0000000000..4e3e816fa5 --- /dev/null +++ b/tests/unit/test_callable_abi_preparation.py @@ -0,0 +1,306 @@ +"""Prepared canonical signatures survive compilation and worker transport.""" + +import inspect +import pickle +from types import SimpleNamespace +from dataclasses import replace +from functools import wraps +from unittest.mock import Mock + +import cloudpickle +import numpy as np +import pytest +from arraybridge import ArrayPayload + +import openhcs.core.callable_contract as contract_module +from openhcs.core.callable_contract import ( + CallableContract, CallableImportIdentity, CallableMetadata, + attach_callable_contract_metadata, attach_processing_prepare, +) +from openhcs.core.function_contract_metadata import FunctionContractAttribute +from openhcs.core.function_patterns import NormalizeFunctionGroupAuthority +from openhcs.core.function_reference import ImportableFunctionReference +from openhcs.core.processing_preparation import CallablePreparation, PreparationCacheBatch +from openhcs.core.runtime_image_values import ImagePayloadMetadata, image_payload_data +from openhcs.processing.backends.lib_registry.registry_service import RegistryService +from openhcs.core.runtime_batch_contracts import runtime_callable_defaults +from openhcs.core.pipeline.function_contracts import resolved_callable_parameter +from openhcs.processing.backends.lib_registry.unified_registry import ( + RuntimeCallablePolicy, RuntimeInvocationKwargPolicy, +) + + +def raw_numeric(image: np.ndarray, scale: float = 2.0) -> np.ndarray: + return image * scale + + +@pytest.fixture +def fake_preparation(monkeypatch): + prepare = Mock() + monkeypatch.setattr(CallablePreparation, "prepare", prepare) + monkeypatch.setattr(PreparationCacheBatch, "populate_child_caches", lambda *a, **k: None) + return prepare + + +def test_registry_resolves_signatures_after_all_hooks_and_before_ready(monkeypatch): + events = [] + + def first(image: np.ndarray): + return image + + def last(image: np.ndarray): + return image + + first.__dict__[FunctionContractAttribute.raw_processing_function] = last + metadata = {"first": SimpleNamespace(func=first), "alias": SimpleNamespace(func=first)} + monkeypatch.setattr(RegistryService, "_metadata_cache", metadata) + monkeypatch.setattr(PreparationCacheBatch, "populate_child_caches", lambda *a, **k: None) + + def prepare(owner): + events.append(("prepare", owner.projection.func)) + if owner.projection.func is last: + last.__annotations__["image"] = ArrayPayload + + original_hints = contract_module.get_type_hints + + def hints(func, **kwargs): + events.append(("signature", func)) + return original_hints(func, **kwargs) + + monkeypatch.setattr(CallablePreparation, "prepare", prepare) + monkeypatch.setattr(contract_module, "get_type_hints", hints) + assert RegistryService.prepare_in_current_process(status_callback=events.append) is metadata + preparations = [i for i, item in enumerate(events) if isinstance(item, tuple) and item[0] == "prepare"] + signatures = [i for i, item in enumerate(events) if isinstance(item, tuple) and item[0] == "signature"] + assert len(preparations) == 2 + assert max(preparations) < min(signatures) + assert max(signatures) < next(i for i, item in enumerate(events) if isinstance(item, str) and "kernels ready" in item) + assert CallableContract.from_callable(first).canonical_parameter_annotations["image"] is ArrayPayload + assert CallableContract.from_callable(last).canonical_parameter_annotations["image"] is ArrayPayload + + +def test_explicit_library_rewarm_refreshes_new_contract_not_existing_snapshot(monkeypatch, fake_preparation): + def raw(image: np.ndarray): + return image + + metadata = {"raw": SimpleNamespace(func=raw)} + monkeypatch.setattr(RegistryService, "_metadata_cache", metadata) + RegistryService.prepare_in_current_process() + first = CallableContract.from_callable(raw) + raw.__annotations__["image"] = ArrayPayload + assert CallableContract.from_callable(raw).canonical_parameter_annotations["image"] is np.ndarray + RegistryService.prepare_in_current_process() + second = CallableContract.from_callable(raw) + assert first.canonical_parameter_annotations["image"] is np.ndarray + assert second.canonical_parameter_annotations["image"] is ArrayPayload + + +def test_authored_compilation_prepares_before_signature_and_refreshes_each_compilation(monkeypatch): + def raw(image: np.ndarray, scale=2): + return image + + desired_annotations = [ArrayPayload, np.ndarray] + + def prepare(owner): + raw.__annotations__["image"] = desired_annotations.pop(0) + + monkeypatch.setattr(CallablePreparation, "prepare", prepare) + first = NormalizeFunctionGroupAuthority().normalize("group", raw).items[0].contract + second = NormalizeFunctionGroupAuthority().normalize("group", raw).items[0].contract + assert first.canonical_parameter_annotations["image"] is ArrayPayload + assert second.canonical_parameter_annotations["image"] is np.ndarray + assert FunctionContractAttribute.canonical_signature not in vars(raw) + + +def test_authored_preparation_precedes_metadata_validation(monkeypatch): + def raw(image: np.ndarray): + return image + events = [] + original = CallableMetadata.from_projection.__func__ + + def validate(cls, projection): + assert events == ["prepared"] + return original(cls, projection) + + monkeypatch.setattr(CallablePreparation, "prepare", lambda self: events.append("prepared")) + monkeypatch.setattr(CallableMetadata, "from_projection", classmethod(validate)) + contract = CallableContract.from_prepared_callable(raw) + assert contract.canonical_parameter_annotations["image"] is np.ndarray + + +def test_unprepared_function_reference_compilation_reads_prepared_live_declaration(monkeypatch, fake_preparation): + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(raw_numeric), + composite_key="test:raw_numeric", + ) + contract = NormalizeFunctionGroupAuthority().normalize("group", reference).items[0].contract + assert contract.func is reference + assert contract.canonical_parameter_annotations["image"] is np.ndarray + assert fake_preparation.call_count == 1 + + +@pytest.mark.parametrize("serializer", [pickle, cloudpickle]) +def test_prepared_signature_metadata_namespace_and_reference_transport(serializer, fake_preparation): + contract = CallableContract.from_prepared_callable(raw_numeric) + metadata = contract.metadata + + def proxy(image): + return image + + proxy.__dict__.update(metadata.as_namespace()) + assert CallableMetadata.from_callable(proxy).canonical_signature == metadata.canonical_signature + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(raw_numeric), + composite_key="test:raw_numeric", metadata=metadata, + ) + restored = serializer.loads(serializer.dumps(reference)) + assert restored.metadata.canonical_signature == metadata.canonical_signature + restored_contract = CallableContract.from_callable(restored) + assert restored_contract.canonical_parameter_annotations["image"] is np.ndarray + assert restored_contract.canonical_signature.parameters["scale"].default == 2.0 + assert restored_contract.canonical_signature.return_annotation is np.ndarray + + +def test_compiled_raw_projection_never_queries_signature_or_annotations(monkeypatch, fake_preparation): + contract = CallableContract.from_prepared_callable(raw_numeric) + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(raw_numeric), + composite_key="test:raw_numeric", metadata=contract.metadata, + ) + restored = CallableContract.from_callable(pickle.loads(pickle.dumps(reference))) + def forbidden(*args, **kwargs): + raise AssertionError("Compiled runtime queried a live signature") + monkeypatch.setattr(contract_module, "get_type_hints", forbidden) + monkeypatch.setattr(inspect, "signature", forbidden) + payload = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with(np.ones((2, 3)), None) + for _ in range(40): + assert restored.raw_main_flow_call_argument(payload) is image_payload_data(payload) + assert restored.primary_input_parameter_name == "image" + assert restored.canonical_parameter_annotations["image"] is np.ndarray + + +def test_custom_signature_and_mutable_default_are_retained_until_recompilation(monkeypatch, fake_preparation): + default = [] + def raw(image, values=None): + return image + raw.__signature__ = inspect.Signature([ + inspect.Parameter("image", inspect.Parameter.POSITIONAL_ONLY, annotation=ArrayPayload), + inspect.Parameter("values", inspect.Parameter.KEYWORD_ONLY, default=default), + ]) + first = CallableContract.from_prepared_callable(raw) + assert first.canonical_signature.parameters["values"].default is default + raw.__signature__ = raw.__signature__.replace(parameters=[ + inspect.Parameter("image", inspect.Parameter.POSITIONAL_ONLY, annotation=np.ndarray), + ]) + second = CallableContract.from_prepared_callable(raw) + assert first.canonical_parameter_annotations["image"] is ArrayPayload + assert second.canonical_parameter_annotations["image"] is np.ndarray + + +@pytest.mark.parametrize("mutation", [ + lambda func: attach_processing_prepare(func, lambda: None), + lambda func: attach_callable_contract_metadata(func, raw_processing_function=raw_numeric), +]) +def test_supported_declaration_changes_invalidate_published_signature(mutation): + def raw(image: np.ndarray): + return image + CallableContract.warm_canonical_signature(raw) + assert FunctionContractAttribute.canonical_signature in vars(raw) + mutation(raw) + assert FunctionContractAttribute.canonical_signature not in vars(raw) + + +def test_malformed_signature_declaration_is_rejected(): + def raw(image): + return image + raw.__dict__[FunctionContractAttribute.canonical_signature] = "not a signature" + with pytest.raises(TypeError, match="inspect.Signature"): + CallableMetadata.from_callable(raw) + + +def test_exact_raw_default_filter_and_type_views_refresh_with_library_snapshot(): + def raw(image: np.ndarray, scale: float = 2): + return image * scale + CallableContract.warm_canonical_signature(raw) + policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) + assert runtime_callable_defaults(raw)["scale"] == 2 + assert resolved_callable_parameter(raw, "scale").annotation is float + assert policy.invocation(raw, (np.ones((2, 3)),), {"scale":3, "unknown":9}).call()[0,0] == 3 + raw.__defaults__ = (4,) + raw.__annotations__["scale"] = int + assert runtime_callable_defaults(raw)["scale"] == 2 + assert resolved_callable_parameter(raw, "scale").annotation is float + CallableContract.warm_canonical_signature(raw) + assert runtime_callable_defaults(raw)["scale"] == 4 + assert resolved_callable_parameter(raw, "scale").annotation is int + + +def test_different_wrapper_signature_remains_live_for_default_filter_and_parameter_queries(): + def raw(image: np.ndarray, private: int = 8): + return image + def wrapper(image: np.ndarray, public: int = 2): + return image * public + wrapper.__dict__[FunctionContractAttribute.raw_processing_function] = raw + CallableContract.warm_canonical_signature(wrapper) + assert runtime_callable_defaults(wrapper) == {"public":2} + assert resolved_callable_parameter(wrapper, "public").annotation is int + policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) + assert policy.invocation(wrapper, (np.ones((2, 3)),), {"public":3, "private":99}).call()[0,0] == 3 + wrapper.__defaults__ = (5,) + assert runtime_callable_defaults(wrapper) == {"public":5} + assert policy.invocation(wrapper, (np.ones((2, 3)),), {"private":99}).call()[0,0] == 5 + + +def test_prepared_type_query_preserves_legacy_nested_annotation_stripping(): + from typing import Annotated, get_type_hints + def raw(image, labels): + return image + raw.__annotations__ = {"image":Annotated[np.ndarray,"image"],"labels":tuple[Annotated[int,"id"],...],"return":Annotated[np.ndarray,"result"]} + expected = get_type_hints(raw) + CallableContract.warm_canonical_signature(raw) + assert resolved_callable_parameter(raw,"image").annotation == expected["image"] + assert resolved_callable_parameter(raw,"labels").annotation == expected["labels"] + assert CallableContract.from_callable(raw).canonical_parameter_annotations["labels"] == raw.__annotations__["labels"] + + +def test_distinct_authored_runtime_signature_transport_and_new_compilation_refresh(fake_preparation): + def raw(image: np.ndarray, scale: int = 2) -> np.ndarray: + return image * scale + @wraps(raw) + def wrapper(*args, **kwargs): + return raw(*args, **kwargs) + wrapper.__signature__ = inspect.signature(raw).replace(parameters=[ + *inspect.signature(raw).parameters.values(), + inspect.Parameter("injected",inspect.Parameter.KEYWORD_ONLY,default=False), + ]) + first = CallableContract.from_prepared_callable(wrapper) + assert "injected" in first.canonical_signature.parameters + assert "injected" not in first.raw_runtime_signature.parameters + assert first.metadata.raw_runtime_signature is not None + restored = pickle.loads(pickle.dumps(first.metadata)) + assert restored.raw_runtime_signature == first.raw_runtime_signature + assert restored.as_namespace()[FunctionContractAttribute.raw_runtime_signature] == first.raw_runtime_signature + assert FunctionContractAttribute.canonical_signature not in vars(raw) + assert FunctionContractAttribute.canonical_signature not in vars(wrapper) + raw.__defaults__ = (4,) + raw.__annotations__["scale"] = float + second = CallableContract.from_prepared_callable(wrapper) + assert first.raw_runtime_signature.parameters["scale"].default == 2 + assert second.raw_runtime_signature.parameters["scale"].default == 4 + assert second.raw_runtime_signature.parameters["scale"].annotation is float + + +def test_same_runtime_layout_reuses_canonical_without_comparing_mutable_defaults(fake_preparation): + class RejectEquality: + def __eq__(self, other): + raise AssertionError("Signature reuse compared a mutable default") + default = RejectEquality() + def raw(image: np.ndarray, values=default) -> np.ndarray: + return image + @wraps(raw) + def wrapper(*args, **kwargs): + return raw(*args, **kwargs) + contract = CallableContract.from_prepared_callable(wrapper) + assert contract.metadata.raw_runtime_signature is None + assert contract.raw_runtime_signature is contract.canonical_signature + assert contract.raw_runtime_signature.parameters["values"].default is default diff --git a/tests/unit/test_cellprofiler_function_contract_execution.py b/tests/unit/test_cellprofiler_function_contract_execution.py index af8e2adff9..8ad2e4dbef 100644 --- a/tests/unit/test_cellprofiler_function_contract_execution.py +++ b/tests/unit/test_cellprofiler_function_contract_execution.py @@ -2,13 +2,16 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import dataclass, replace from functools import wraps import inspect +from typing import Annotated, Any, Union import numpy as np import pytest +from arraybridge import ArrayPayload +import openhcs.core.callable_contract as callable_contract_module from openhcs.core.aligned_image_payload import ( AlignedImageStack, @@ -35,14 +38,22 @@ from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, + image_payload_mask, + image_payload_metadata, ) +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.source_spatial_domain import SourceSpatialDomain from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( CellProfilerFunctionContractExecutor, ) +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter from openhcs.processing.backends.cellprofiler.alignment import AlignShiftMeasurement -from openhcs.processing.backends.cellprofiler.morphology import morphologicalskeleton +from openhcs.processing.backends.cellprofiler.morphology import ( + MorphOperation, + morph, + morphologicalskeleton, +) from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract @@ -62,7 +73,7 @@ def _compiled_contract( artifact_inputs=artifact_inputs, artifact_outputs=artifact_outputs, ), - ) + ).with_prepared_signature() def test_morphological_skeleton_executes_one_planar_runtime_image() -> None: @@ -83,6 +94,431 @@ def test_morphological_skeleton_executes_one_planar_runtime_image() -> None: assert image_payload_data(result).shape == image.shape +def test_registered_morph_distance_projects_raw_abi_and_retains_source_context() -> None: + contract = CallableContract.from_callable(morph) + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + artifact_outputs=(ArtifactSpec.output("Distance", ImageArtifactType),), + ), + ) + pixels = np.zeros((1, 7, 7), dtype=np.float32) + pixels[:, 2:5, 2:5] = 1 + mask = np.ones_like(pixels, dtype=bool) + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_names=("SavedLabels",), + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/tmp/saved-labels.tif",), + component_metadata=({"channel": "2", "well": "A01"},), + ), + ).payload_with(pixels, mask) + expected = contract.resolve_raw_runtime_callable()( + pixels[0], operation=MorphOperation.DISTANCE + ) + + result = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + {"operation": MorphOperation.DISTANCE}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ), + ) + + np.testing.assert_array_equal(image_payload_data(result), expected[None]) + np.testing.assert_array_equal(image_payload_mask(result), mask) + metadata = image_payload_metadata(result) + assert metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert metadata.source_image_names == ("SavedLabels",) + assert metadata.source_image_provenance_planes.paths == ("/tmp/saved-labels.tif",) + assert metadata.source_image_provenance_planes.component_metadata == ( + {"channel": "2", "well": "A01"}, + ) + + +@pytest.mark.parametrize("payload_annotation", (False, True)) +def test_raw_slice_abi_retains_declared_payloads_and_projects_kwargs( + payload_annotation: bool, +) -> None: + seen = [] + + def array_raw(image: np.ndarray, *, labels: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + seen.append((image, labels)) + return image + labels + + def payload_raw(image: ArrayPayload, *, labels: np.ndarray) -> ArrayPayload: + assert isinstance(image, ArrayPayload) + seen.append((image, labels)) + return image_payload_metadata(image).payload_with( + image_payload_data(image) + labels, image_payload_mask(image), + ) + + raw = payload_raw if payload_annotation else array_raw + contract = _compiled_contract(raw, ProcessingContract.PURE_2D) + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4), dtype=np.float32), np.ones((3, 4), dtype=bool), + ) + selected_labels = np.full((3, 4), 5, dtype=np.float32) + executor = CellProfilerFunctionContractExecutor( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=2, + ), + ) + + assert contract.runtime_main_flow_call_argument(source) is source + result = executor.execute_pure_2d_slice( + contract, raw, source, + {"labels": RuntimeSliceAlignedValues(slices=(np.zeros((3, 4)), selected_labels)), + "adapter_only_control": 1}, + 1, 2, + ) + + assert seen[0][0] is (source if payload_annotation else image_payload_data(source)) + assert seen[0][1] is selected_labels + np.testing.assert_array_equal(image_payload_data(result), np.full((3, 4), 6)) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(source)) + assert image_payload_metadata(result).source_image_names == ("DNA",) + + +def test_raw_slice_abi_does_not_relax_output_mask_shape_validation() -> None: + def bad_shape(image: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + return image[:1] + + contract = _compiled_contract(bad_shape, ProcessingContract.PURE_2D) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4)), np.ones((3, 4), dtype=bool), + ) + executor = CellProfilerFunctionContractExecutor( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ), + ) + with pytest.raises(ValueError, match="[Mm]ask.*shape|shape.*[Mm]ask"): + executor.execute_pure_2d_slice(contract, bad_shape, source, {}, 0, 1) + + +@pytest.mark.parametrize("payload_annotation", (False, True)) +@pytest.mark.parametrize( + ("processing_contract", "execution_mode"), + ( + (ProcessingContract.PURE_2D, ImagePayloadExecutionMode.NATURAL), + (ProcessingContract.PURE_3D, ImagePayloadExecutionMode.NATURAL), + (ProcessingContract.PURE_2D, ImagePayloadExecutionMode.FULL_STACK), + (ProcessingContract.PURE_3D, ImagePayloadExecutionMode.FULL_STACK), + ), +) +def test_raw_abi_projection_matches_declared_argument_in_single_and_full_stack_modes( + payload_annotation: bool, + processing_contract: ProcessingContract, + execution_mode: ImagePayloadExecutionMode, +) -> None: + seen = [] + + def array_raw(image: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + seen.append(image) + return image + + def payload_raw(image: ArrayPayload) -> ArrayPayload: + assert isinstance(image, ArrayPayload) + seen.append(image) + return image + + raw = payload_raw if payload_annotation else array_raw + contract = _compiled_contract(raw, processing_contract) + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + artifact_outputs=(ArtifactSpec.output("Result", ImageArtifactType),), + ), + ) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4)), None, + ) + + result = CellProfilerFunctionContractExecutor().execute( + contract, raw, source, {}, execution_mode=execution_mode, + ) + + assert seen == [source if payload_annotation else image_payload_data(source)] + assert seen[0] is (source if payload_annotation else image_payload_data(source)) + assert result is seen[0] + + +@pytest.mark.parametrize( + ("annotation", "retains_payload"), + ( + (np.ndarray, False), + (ArrayPayload, True), + (RuntimeArrayData, True), + (Union[ArrayPayload, np.ndarray], True), + (Annotated[ArrayPayload, "source context"], True), + (Annotated[RuntimeArrayData, "source context"], True), + (Union[Annotated[ArrayPayload, "source context"], np.ndarray], True), + (Union[Annotated[RuntimeArrayData, "source context"], None], True), + (object, True), + (Any, False), + (inspect.Parameter.empty, False), + (Sequence[ArrayPayload], False), + (tuple[ArrayPayload, ...], False), + ), +) +def test_canonical_argument_projection_uses_declared_nominal_annotation( + annotation: object, retains_payload: bool, +) -> None: + def raw(image): + return image + + if annotation is not inspect.Parameter.empty: + raw.__annotations__["image"] = annotation + contract = _compiled_contract(raw, ProcessingContract.PURE_2D) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + np.ones((3, 4)), None, + ) + expected = source if retains_payload else image_payload_data(source) + + assert contract.raw_main_flow_call_argument(source) is expected + assert contract.runtime_main_flow_call_argument(source) is expected + adapted = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + assert adapted.raw_main_flow_call_argument(source) is expected + assert adapted.runtime_main_flow_call_argument(source) is source + + +def test_raw_slice_argument_retains_buffer_identity_for_inplace_array_callable() -> None: + def inplace(image: np.ndarray) -> np.ndarray: + image[0, 0] = 7 + return image + + contract = _compiled_contract(inplace, ProcessingContract.PURE_2D) + pixels = np.zeros((3, 4)) + source = ImagePayloadMetadata(source_image_names=("DNA",)).payload_with( + pixels, None, + ) + executor = CellProfilerFunctionContractExecutor( + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ), + ) + result = executor.execute_pure_2d_slice(contract, inplace, source, {}, 0, 1) + + assert image_payload_data(result) is pixels + assert pixels[0, 0] == 7 + + +def test_prepared_raw_abi_performs_no_introspection_until_explicit_refresh( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen = [] + + def raw(image: np.ndarray) -> np.ndarray: + seen.append(isinstance(image, ArrayPayload)) + return image_payload_data(image) + + contract = _compiled_contract( + raw, ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result", ImageArtifactType),), + ) + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, source_image_names=("DNA",), + ).payload_with(np.ones((12, 3, 4)), None) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=12, + ) + executor = CellProfilerFunctionContractExecutor(plane_projection=projection) + hints = [] + signatures = [] + original_hints = callable_contract_module.get_type_hints + original_signature = inspect.signature + + def counted_hints(func, **kwargs): + hints.append((func, kwargs)) + return original_hints(func, **kwargs) + + def counted_signature(func, **kwargs): + caller = inspect.currentframe().f_back + if caller.f_globals is callable_contract_module.__dict__: + signatures.append(caller.f_code.co_name) + return original_signature(func, **kwargs) + + monkeypatch.setattr(callable_contract_module, "get_type_hints", counted_hints) + monkeypatch.setattr(inspect, "signature", counted_signature) + + for annotation in (np.ndarray, ArrayPayload): + raw.__annotations__["image"] = annotation + contract = contract.with_prepared_signature() + result = executor.execute( + contract, raw, source, {}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=projection, + ) + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(source)) + + assert seen == [False] * 12 + [True] * 12 + assert hints == [(raw, {"include_extras": True})] * 2 + assert signatures == ["resolve_signature"] * 2 + assert not hasattr(executor, "_raw_argument_types") + + +def test_raw_abi_derivation_follows_existing_input_guards(monkeypatch) -> None: + def raw(image: np.ndarray) -> np.ndarray: + raise AssertionError("Invalid image reached raw execution") + + contract = _compiled_contract(raw, ProcessingContract.PURE_2D) + + def reject_premature_annotations(self): + raise AssertionError("Input guard resolved the raw ABI prematurely") + + monkeypatch.setattr( + CallableContract, "canonical_parameter_declarations", + property(reject_premature_annotations), + ) + with pytest.raises(TypeError, match="requires AlignedImageStack"): + CellProfilerFunctionContractExecutor().execute( + contract, raw, np.zeros((2, 3)), {}, + execution_mode=ImagePayloadExecutionMode.ALIGNED_MULTI_IMAGE_STACK, + ) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=2, + ) + mismatched_image = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + ).payload_with(np.zeros((2, 3, 4)), None) + with pytest.raises(RuntimeSliceProjectionDeclarationError, match="conflicts"): + CellProfilerFunctionContractExecutor().execute( + contract, raw, mismatched_image, {}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=projection, + ) + + +def test_raw_abi_child_preserves_cooperative_executor_constructor_contract() -> None: + constructed = [] + raw_calls = [] + + class CooperativeExecutor(CellProfilerFunctionContractExecutor): + def __init__(self, plane_projection=None): + super().__init__(plane_projection=plane_projection) + constructed.append(self) + + def invoke_raw(self, callable_contract, func, image, kwargs): + raw_calls.append(self) + return super().invoke_raw(callable_contract, func, image, kwargs) + + def raw(image: np.ndarray) -> np.ndarray: + assert isinstance(image, np.ndarray) + return image + + contract = _compiled_contract( + raw, ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result", ImageArtifactType),), + ) + source = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE).payload_with( + np.ones((2, 3, 4)), None, + ) + executor = CooperativeExecutor() + result = executor.execute( + contract, raw, source, {}, execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=2, + ), + ) + + assert len(constructed) == 2 + assert raw_calls == [constructed[1]] * 2 + assert not hasattr(executor, "_raw_argument_types") + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(source)) + + def foreign_raw(image: ArrayPayload) -> ArrayPayload: + assert isinstance(image, ArrayPayload) + return image + + foreign_contract = _compiled_contract(foreign_raw, ProcessingContract.PURE_2D) + child = constructed[1] + assert child.invoke_raw(foreign_contract, foreign_raw, source, {}) is source + assert not hasattr(child, "_scope_contract") + + +def test_actual_prepared_executor_planes_make_no_signature_or_hint_queries(monkeypatch) -> None: + calls = [] + def raw(image: np.ndarray, scale: int = 2) -> np.ndarray: + calls.append(image.shape) + return image * scale + CallableContract.warm_canonical_signature(raw) + contract = _compiled_contract( + raw, ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result",ImageArtifactType),), + ) + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE,source_image_names=("DNA",), + ).payload_with(np.ones((12,3,4)),None) + projection = RuntimePlaneAxisValueProjection.preserve(axis=RuntimePlaneAxis.RUNTIME_SLICE,axis_size=12) + def forbidden(*args, **kwargs): + raise AssertionError("Prepared execution introspected a live callable") + monkeypatch.setattr(inspect,"signature",forbidden) + monkeypatch.setattr(callable_contract_module,"get_type_hints",forbidden) + result = CellProfilerFunctionContractExecutor().execute( + contract,raw,source,{},execution_mode=ImagePayloadExecutionMode.NATURAL, + plane_projection=projection, + ) + np.testing.assert_array_equal(image_payload_data(result),np.full((12,3,4),2)) + assert calls == [(3,4)]*12 + + +def test_authored_distinct_raw_abi_batch_has_no_runtime_queries_and_filters_controls(monkeypatch) -> None: + from openhcs.core.processing_preparation import CallablePreparation + calls = [] + def raw(image: np.ndarray, scale: int = 2) -> np.ndarray: + calls.append(image.shape) + return image * scale + @wraps(raw) + def wrapper(*args, **kwargs): + return raw(*args, **kwargs) + wrapper.__signature__ = inspect.signature(raw).replace(parameters=[ + *inspect.signature(raw).parameters.values(), + inspect.Parameter("injected",inspect.Parameter.KEYWORD_ONLY,default=False), + ]) + monkeypatch.setattr(CallablePreparation,"prepare",lambda self:None) + contract = CallableContract.from_prepared_callable(wrapper) + contract = replace(contract,metadata=replace( + contract.metadata,processing_contract=ProcessingContract.PURE_2D, + artifact_outputs=(ArtifactSpec.output("Result",ImageArtifactType),), + )) + source = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE,source_image_names=("DNA",)).payload_with(np.ones((12,3,4)),None) + projection = RuntimePlaneAxisValueProjection.preserve(axis=RuntimePlaneAxis.RUNTIME_SLICE,axis_size=12) + def forbidden(*args, **kwargs): + raise AssertionError("Prepared authored raw target queried a live signature") + monkeypatch.setattr(inspect,"signature",forbidden) + monkeypatch.setattr(callable_contract_module,"get_type_hints",forbidden) + result = CellProfilerFunctionContractExecutor().execute( + contract,wrapper,source,{"injected":True}, + execution_mode=ImagePayloadExecutionMode.NATURAL,plane_projection=projection, + ) + np.testing.assert_array_equal(image_payload_data(result),np.full((12,3,4),2)) + assert calls == [(3,4)]*12 + + def test_compiled_slice_execution_resolves_raw_target_once_and_preserves_request_binding( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -103,7 +539,7 @@ def request_bound(request: ScaleRequest) -> object: def decorated(*args: object, **kwargs: object) -> object: raise AssertionError("CellProfiler invoked an outer runtime wrapper") - contract = CallableContract.from_callable(decorated) + contract = CallableContract.from_callable(decorated).with_prepared_signature() contract = replace( contract, metadata=replace( diff --git a/tests/unit/test_cellprofiler_prepared_payload_abi.py b/tests/unit/test_cellprofiler_prepared_payload_abi.py new file mode 100644 index 0000000000..bf0f122d11 --- /dev/null +++ b/tests/unit/test_cellprofiler_prepared_payload_abi.py @@ -0,0 +1,127 @@ +"""Compiled raw calls preserve metadata required by declared image consumers.""" + +from dataclasses import replace + +import numpy as np +from scipy import ndimage + +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.function_patterns import NormalizeFunctionGroupAuthority +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, + normalize_image_payload_intensity, +) +from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest +from openhcs.core.runtime_object_labels import object_label_dense_array +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter +from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( + CellProfilerFunctionContractExecutor, +) +from openhcs.processing.backends.cellprofiler.color import ColorToGrayMode, color_to_gray +from openhcs.processing.backends.cellprofiler.median_filter import medianfilter +from openhcs.processing.backends.cellprofiler.secondary import ( + SecondaryMethod, + identify_secondary_objects, +) + + +def _execute_prepared(func, source, kwargs, mode=ImagePayloadExecutionMode.NATURAL): + contract = NormalizeFunctionGroupAuthority().normalize("image", func).items[0].contract + contract = replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + assert contract.metadata.canonical_signature is not None + assert contract.raw_main_flow_call_argument(source) is source + return CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + kwargs, + execution_mode=mode, + ) + + +def test_prepared_color_to_gray_preserves_channel_mask_and_spatial_domain(): + pixels = np.arange(4 * 5 * 3, dtype=np.float32).reshape(4, 5, 3) + mask = np.ones(pixels.shape[:2], dtype=bool) + mask[0] = False + mask[-1] = False + domain = SourceSpatialDomain(origin_yx=(2, 3), source_shape_yx=(8, 10)) + source = ImagePayloadMetadata( + source_channel_axis=-1, + source_image_names=("RGB",), + source_path="/input/rgb.tif", + source_spatial_domain=domain, + ).payload_with(pixels, mask) + + result = _execute_prepared( + color_to_gray, source, + {"mode": ColorToGrayMode.COMBINE, + "channel_indices": (0, 2), "contributions": (1.0, 3.0)}, + ) + + np.testing.assert_array_equal(image_payload_data(result), (pixels[..., 0] + 3 * pixels[..., 2]) / 4) + np.testing.assert_array_equal(image_payload_mask(result), mask) + metadata = image_payload_metadata(result) + assert metadata.source_channel_axis is None + assert metadata.source_spatial_domain == domain + assert metadata.source_image_paths == ("/input/rgb.tif",) + + +def test_prepared_medianfilter_keeps_original_intensity_scale_and_mask(): + pixels = np.arange(25, dtype=np.uint16).reshape(5, 5) + mask = np.ones(pixels.shape, dtype=bool) + mask[0] = False + source = normalize_image_payload_intensity( + ImagePayloadMetadata.for_array(pixels).payload_with(pixels, mask), + dtype=np.float32, + ) + + result = _execute_prepared( + medianfilter, source, {"window_size": 3}, ImagePayloadExecutionMode.FULL_STACK, + ) + + np.testing.assert_array_equal( + image_payload_data(result), + ndimage.median_filter(image_payload_data(source), size=3, mode="constant"), + ) + np.testing.assert_array_equal(image_payload_mask(result), mask) + assert image_payload_metadata(result).unit_interval_intensity_scale == 65535 + assert image_payload_data(result).dtype == np.float32 + + +def test_prepared_secondary_object_output_keeps_actual_source_spatial_domain(): + pixels = np.zeros((9, 9), dtype=np.float32) + domain = SourceSpatialDomain(origin_yx=(3, 4), source_shape_yx=(15, 16)) + source = ImagePayloadMetadata( + source_image_names=("DNA",), source_path="/input/dna.tif", + source_spatial_domain=domain, + ).payload_with(pixels, None) + seed_pixels = np.zeros(pixels.shape, dtype=np.int32) + seed_pixels[4, 4] = 1 + seeds = SourceImageObjectLabelBuildRequest( + image=source, labels=seed_pixels, declared_object_ids=(1,), + ).payload() + + result = _execute_prepared( + identify_secondary_objects, source, + {"primary_labels": seeds, "method": SecondaryMethod.DISTANCE_N, + "distance_to_dilate": 1, "fill_holes": False}, + ) + + _image, _rows, relationship, objects = result + expected = ndimage.distance_transform_edt(seed_pixels == 0) <= 1 + np.testing.assert_array_equal(object_label_dense_array(objects), expected.astype(np.int32)) + assert objects.source_spatial_domain.origin_yx == domain.origin_yx + assert objects.source_spatial_domain.source_shape_yx == domain.source_shape_yx + assert objects.source_provenance == image_payload_metadata(source).source_provenance + assert relationship.source_ids == (1,) + assert relationship.target_ids == (1,) diff --git a/tests/unit/test_preparation_admission.py b/tests/unit/test_preparation_admission.py index 3ed203e53d..99c99a3a55 100644 --- a/tests/unit/test_preparation_admission.py +++ b/tests/unit/test_preparation_admission.py @@ -189,8 +189,10 @@ def prepare_selected(): ) dynamic_invocation = CompiledFunctionInvocation( key=FunctionInvocationKey("dynamic", "default", 1), - contract=CallableContract.from_callable(dynamic), + contract=CallableContract.from_prepared_callable(dynamic), ) + assert events == ["selected", "unselected", "dynamic"] + assert dynamic_invocation.contract.metadata.canonical_signature is not None group = CompiledFunctionGroup("default", (invocation, dynamic_invocation)) pattern = CompiledFunctionPattern(groups=(group,), is_grouped=False) context = SimpleNamespace(step_plans={0: SimpleNamespace(step_index=0, compiled_function_pattern=pattern)}) From 760e9ae8613d8737411a627b6abfe6a15f4ca0b9 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 01:42:38 -0400 Subject: [PATCH 040/125] Keep prepared callable ABI algorithms on metadata and invocation owners --- openhcs/core/callable_contract.py | 310 +++++++++--------- openhcs/core/runtime_batch_contracts.py | 8 + .../runtime/function_contract_execution.py | 49 +-- .../backends/lib_registry/registry_service.py | 4 +- .../backends/lib_registry/unified_registry.py | 20 +- tests/unit/test_callable_abi_preparation.py | 46 ++- ...ellprofiler_function_contract_execution.py | 13 +- 7 files changed, 240 insertions(+), 210 deletions(-) diff --git a/openhcs/core/callable_contract.py b/openhcs/core/callable_contract.py index d462097d29..5d0640ef6a 100644 --- a/openhcs/core/callable_contract.py +++ b/openhcs/core/callable_contract.py @@ -316,6 +316,125 @@ def callable_signature(cls, func: Callable[..., object]) -> inspect.Signature: signature = cls.prepared_callable_signature(func) return inspect.signature(func) if signature is None else signature + def resolve_canonical_raw_callable(self, func: Any) -> Callable[..., object]: + """Return the declaration-owned callable whose signature defines behavior.""" + + declared = self.raw_processing_function + if declared is None: + declared = func + return _resolve_declared_callable(declared) + + def resolve_raw_runtime_callable(self, func: Any) -> Callable[..., object]: + """Remove runtime wrappers without crossing a semantic request binding.""" + + canonical = self.resolve_canonical_raw_callable(func) + if self.request_binding is None: + return inspect.unwrap(canonical) + + binding_key = FunctionContractAttribute.callable_request_binding + + def is_request_boundary(candidate: Callable[..., object]) -> bool: + namespace = _callable_namespace(candidate) + if binding_key not in namespace: + return False + wrapped = namespace.get("__wrapped__") + return not callable(wrapped) or binding_key not in _callable_namespace( + wrapped + ) + + resolved = inspect.unwrap(canonical, stop=is_request_boundary) + if binding_key not in _callable_namespace(resolved): + raise RuntimeError( + f"Callable {CallableProjection.from_callable(func).name!r} declares a request binding, but " + "its canonical raw callable has no request-binding boundary." + ) + return resolved + + def resolve_signature(self, raw_callable: Callable[..., object]) -> inspect.Signature: + """Resolve the semantic ABI once after declaration preparation.""" + signature = inspect.signature(raw_callable) + annotations = ( + self.request_binding.public_annotations_dict + if self.request_binding is not None + else get_type_hints(raw_callable, include_extras=True) + ) + return signature.replace( + parameters=tuple( + parameter.replace(annotation=annotations.get(name, parameter.annotation)) + for name, parameter in signature.parameters.items() + ), + return_annotation=annotations.get("return", signature.return_annotation), + ) + + @staticmethod + def primary_input_name(parameters: Mapping[str, inspect.Parameter]) -> str | None: + for parameter in parameters.values(): + if parameter.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ): + return parameter.name + return None + + @classmethod + def _nominal_argument_types(cls, annotation: object) -> tuple[type, ...]: + annotation = resolve_annotated(annotation) + if is_union_type(annotation): + return tuple( + argument_type for member in get_args(annotation) + for argument_type in cls._nominal_argument_types(member) + ) + return ( + (annotation,) + if annotation is not Any and isinstance(annotation, type) + else () + ) + + def raw_main_flow_call_argument(self, source_payload: Any, signature: inspect.Signature) -> Any: + """Admit a declared nominal carrier at the canonical image boundary.""" + from openhcs.core.runtime_image_values import image_payload_data + + primary_name = self.primary_input_name(signature.parameters) + annotation = None if primary_name is None else signature.parameters[primary_name].annotation + return source_payload if any( + isinstance(source_payload, argument_type) + for argument_type in self._nominal_argument_types(annotation) + ) else image_payload_data(source_payload) + + def canonical_signature_for(self, func: Any) -> inspect.Signature: + """Use the prepared semantic ABI; unprepared authoring remains live.""" + if self.canonical_signature is not None: + return self.canonical_signature + return self.resolve_signature(self.resolve_canonical_raw_callable(func)) + + def raw_runtime_signature_for(self, func: Any) -> inspect.Signature: + """Own the distinct raw ABI or the shared canonical target snapshot.""" + if self.raw_runtime_signature is not None: + return self.raw_runtime_signature + if self.canonical_signature is not None: + return self.canonical_signature + return self.resolve_signature(self.resolve_raw_runtime_callable(func)) + + def with_prepared_signatures(self, canonical: Callable, runtime: Callable) -> "CallableMetadata": + """Snapshot each actual target independently after declaration preparation.""" + return dataclasses.replace( + self, canonical_signature=self.resolve_signature(canonical), + raw_runtime_signature=None if runtime is canonical else self.resolve_signature(runtime), + ) + + def with_missing_signatures(self, canonical: Callable[[], Callable], runtime: Callable[[], Callable]) -> "CallableMetadata": + """Reuse library readiness or capture the newly prepared authored targets.""" + if self.canonical_signature is not None: + return self + return self.with_prepared_signatures(canonical(), runtime()) + + def publish_signatures(self, namespace: MutableMapping[str, Any]) -> None: + """Replace the namespace pair as one library preparation epoch.""" + namespace[FunctionContractAttribute.canonical_signature] = self.canonical_signature + namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) + if self.raw_runtime_signature is not None: + namespace[FunctionContractAttribute.raw_runtime_signature] = self.raw_runtime_signature + @property def artifact_output_policy(self) -> type[ArtifactOutputPolicy]: """Project recording ownership from this callable's adapter declaration.""" @@ -959,43 +1078,9 @@ def runtime_main_flow_call_argument(self, source_payload: Any) -> Any: return source_payload return self.raw_main_flow_call_argument(source_payload) - def raw_main_flow_call_argument( - self, source_payload: Any, - ) -> Any: - """Project one image argument at the canonical raw callable boundary.""" - from openhcs.core.runtime_image_values import image_payload_data - - return ( - source_payload - if any( - isinstance(source_payload, argument_type) - for argument_type in self.raw_main_flow_argument_types() - ) - else image_payload_data(source_payload) - ) - - def raw_main_flow_argument_types(self) -> tuple[type, ...]: - """Derive the nominal argument view for one canonical invocation.""" - parameters = self.canonical_parameter_declarations - primary_name = self._primary_input_name(parameters) - annotation = ( - None if primary_name is None else parameters[primary_name].annotation - ) - return self._nominal_argument_types(annotation) - - @classmethod - def _nominal_argument_types(cls, annotation: object) -> tuple[type, ...]: - annotation = resolve_annotated(annotation) - if is_union_type(annotation): - return tuple( - argument_type for member in get_args(annotation) - for argument_type in cls._nominal_argument_types(member) - ) - return ( - (annotation,) - if annotation is not Any and isinstance(annotation, type) - else () - ) + def raw_main_flow_call_argument(self, source_payload: Any) -> Any: + """Project the metadata-owned raw carrier declaration for this contract.""" + return self.metadata.raw_main_flow_call_argument(source_payload, self.canonical_signature) def main_flow_call_argument(self, source_payload: Any) -> Any: """Let the processing declaration retain context needed before raw calls.""" @@ -1064,17 +1149,7 @@ def artifact_specs(self) -> ArtifactSpecCollection: def primary_input_parameter_name(self) -> str | None: """FunctionStep input payload parameter declared by callable signature.""" signature = self.canonical_signature - return self._primary_input_name(signature.parameters) - - @staticmethod - def _primary_input_name(parameters: Mapping[str, inspect.Parameter]) -> str | None: - for parameter in parameters.values(): - if parameter.kind in ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - ): - return parameter.name - return None + return self.metadata.primary_input_name(signature.parameters) @property def accepts_implicit_main_flow_input(self) -> bool: @@ -1112,12 +1187,8 @@ def resolve_runtime_callable(self) -> Callable[..., object]: return runtime_adapter.executable_callable(resolved, self) def resolve_canonical_raw_callable(self) -> Callable[..., object]: - """Return the declaration-owned callable whose signature defines behavior.""" - - declared = self.raw_processing_function - if declared is None: - declared = self.func - return _resolve_declared_callable(declared) + """Return the declaration-owned semantic target.""" + return self.metadata.resolve_canonical_raw_callable(self.func) def canonical_raw_import_identity(self) -> CallableImportIdentity: """Return the complete import identity of the canonical raw callable.""" @@ -1127,30 +1198,8 @@ def canonical_raw_import_identity(self) -> CallableImportIdentity: ) def resolve_raw_runtime_callable(self) -> Callable[..., object]: - """Remove runtime wrappers without crossing a semantic request binding.""" - - canonical = self.resolve_canonical_raw_callable() - if self.request_binding is None: - return inspect.unwrap(canonical) - - binding_key = FunctionContractAttribute.callable_request_binding - - def is_request_boundary(candidate: Callable[..., object]) -> bool: - namespace = _callable_namespace(candidate) - if binding_key not in namespace: - return False - wrapped = namespace.get("__wrapped__") - return not callable(wrapped) or binding_key not in _callable_namespace( - wrapped - ) - - resolved = inspect.unwrap(canonical, stop=is_request_boundary) - if binding_key not in _callable_namespace(resolved): - raise RuntimeError( - f"Callable {self.function_name!r} declares a request binding, but " - "its canonical raw callable has no request-binding boundary." - ) - return resolved + """Resolve the metadata-owned raw target without crossing request binding.""" + return self.metadata.resolve_raw_runtime_callable(self.func) @property def canonical_parameter_annotations(self) -> Mapping[str, object]: @@ -1167,88 +1216,30 @@ def canonical_parameter_declarations(self) -> Mapping[str, inspect.Parameter]: @property def canonical_signature(self) -> inspect.Signature: - """Use the prepared declaration, or query an uncompiled authoring callable.""" - if self.metadata.canonical_signature is not None: - return self.metadata.canonical_signature - return self.resolve_signature(self.resolve_canonical_raw_callable()) - - def resolve_signature(self, raw_callable: Callable[..., object]) -> inspect.Signature: - """Resolve the semantic ABI once after declaration preparation.""" - signature = inspect.signature(raw_callable) - annotations = ( - self.request_binding.public_annotations_dict - if self.request_binding is not None - else get_type_hints(raw_callable, include_extras=True) - ) - return signature.replace( - parameters=tuple( - parameter.replace(annotation=annotations.get(name, parameter.annotation)) - for name, parameter in signature.parameters.items() - ), - return_annotation=annotations.get("return", signature.return_annotation), - ) + """The metadata-owned semantic signature for this contract's target.""" + return self.metadata.canonical_signature_for(self.func) def with_prepared_signature(self) -> "CallableContract": - """Capture the prepared ABI on the declaration transported to workers.""" - canonical = self.resolve_canonical_raw_callable() - signature = self.resolve_signature(canonical) - runtime = self.resolve_raw_runtime_callable() - runtime_signature = None - if runtime is not canonical: - candidate = self.resolve_signature(runtime) - canonical_parameters = tuple(signature.parameters.values()) - runtime_parameters = tuple(candidate.parameters.values()) - same_parameters = len(canonical_parameters) == len(runtime_parameters) and all( - left.name == right.name and left.kind is right.kind - and left.default is right.default and left.annotation is right.annotation - for left, right in zip(canonical_parameters, runtime_parameters) - ) - if not (same_parameters and signature.return_annotation is candidate.return_annotation): - runtime_signature = candidate - return dataclasses.replace( - self, - metadata=dataclasses.replace( - self.metadata, canonical_signature=signature, - raw_runtime_signature=runtime_signature, - ), - ) + """Freeze the prepared declaration into its existing compiled contract.""" + return dataclasses.replace(self, metadata=self.metadata.with_prepared_signatures( + self.resolve_canonical_raw_callable(), self.resolve_raw_runtime_callable(), + )) @property def raw_runtime_signature(self) -> inspect.Signature: - """The exact unwrapped execution ABI, preserving request boundaries.""" - if self.metadata.raw_runtime_signature is not None: - return self.metadata.raw_runtime_signature - if self.metadata.canonical_signature is not None: - return self.metadata.canonical_signature - return self.resolve_signature(self.resolve_raw_runtime_callable()) - - @classmethod - def warm_canonical_signature(cls, func: Callable[..., object]) -> None: - """Publish one refreshed ABI after all library preparation hooks finish.""" - contract = cls.from_callable(func).with_prepared_signature() - _mutable_callable_namespace(func)[FunctionContractAttribute.canonical_signature] = ( - contract.metadata.canonical_signature - ) - namespace = _mutable_callable_namespace(func) - namespace.pop(FunctionContractAttribute.raw_runtime_signature, None) - if contract.metadata.raw_runtime_signature is not None: - namespace[FunctionContractAttribute.raw_runtime_signature] = contract.metadata.raw_runtime_signature + """The metadata-owned execution signature for this contract's target.""" + return self.metadata.raw_runtime_signature_for(self.func) @classmethod def from_prepared_callable(cls, func: Callable[..., object]) -> "CallableContract": - """Reuse library warmup; prepare authored declarations before validation.""" + """Construct a compiled view only after the declaration owner prepares it.""" projection = CallableProjection.from_callable(func) - if projection.namespace.get(FunctionContractAttribute.canonical_signature) is not None: - return cls.from_callable(func) - resolved = _resolve_declared_callable(func) - for target in CallableProjection.from_callable(resolved).prepare_targets(): - prepare_processing_callable(target) - prepared = cls.from_callable(resolved) - if prepared.metadata.canonical_signature is None: - prepared = prepared.with_prepared_signature() + prepared = cls.from_callable(projection.prepared_callable()) return dataclasses.replace( - prepared, func=func, function_name=projection.name, - module_name=projection.module_name, + prepared, func=func, function_name=projection.name, module_name=projection.module_name, + metadata=prepared.metadata.with_missing_signatures( + prepared.resolve_canonical_raw_callable, prepared.resolve_raw_runtime_callable, + ), ) def decode_public_kwargs(self, kwargs: Mapping[str, object]) -> dict[str, object]: @@ -1951,6 +1942,21 @@ def from_callable(cls, func: Any) -> "CallableProjection": ) return cls(func=func, name=name, module_name=module_name, namespace=namespace) + def prepared_callable(self) -> Any: + """Keep ready library references or prepare live authored targets before validation.""" + reader = CallableMetadataReader(self.namespace, self.name) + if reader.has_prepared_signature(): + return self.func + resolved = _resolve_declared_callable(self.func) + for target in type(self).from_callable(resolved).prepare_targets(): + prepare_processing_callable(target) + return resolved + + def warm_canonical_signature(self) -> None: + """Publish prepared target signatures only after all library hooks finish.""" + contract = CallableContract.from_callable(self.func).with_prepared_signature() + contract.metadata.publish_signatures(_mutable_callable_namespace(self.func)) + def prepare_targets(self) -> tuple[Any, ...]: """Return wrapper/raw callables that may carry preparation metadata.""" targets: list[Any] = [] @@ -1996,6 +2002,10 @@ class CallableMetadataReader: namespace: CallableNamespace function_name: str + def has_prepared_signature(self) -> bool: + """Validate the namespace-owned readiness declaration before phase selection.""" + return self.optional_signature(FunctionContractAttribute.canonical_signature) is not None + def optional_signature(self, field_name: str) -> inspect.Signature | None: value = self.namespace.get(field_name) if value is not None and not isinstance(value, inspect.Signature): diff --git a/openhcs/core/runtime_batch_contracts.py b/openhcs/core/runtime_batch_contracts.py index cf48ff9032..d3eef11fcc 100644 --- a/openhcs/core/runtime_batch_contracts.py +++ b/openhcs/core/runtime_batch_contracts.py @@ -280,6 +280,14 @@ def executors(self) -> Mapping[RuntimeBatchExecutionDomain, Callable]: class Pure2DSliceBatchExecutor(RuntimeBatchExecutor): """Base contract for equivalent pure-2D slice batch execution.""" + @classmethod + def from_executors( + cls, executors: Mapping[RuntimeBatchExecutionDomain, Callable] | None, + ) -> Callable: + """Honor a declared pure-2D executor before the nominal serial default.""" + declared = None if executors is None else executors.get(RuntimeBatchExecutionDomain.PURE_2D_SLICES) + return declared if callable(declared) else cls.default_executor() + @classmethod def default_executor(cls) -> "Pure2DSliceBatchExecutor": """Return the single-thread default pure-2D batch executor.""" diff --git a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py index 76ab77ecca..199f2dea38 100644 --- a/openhcs/interop/cellprofiler/runtime/function_contract_execution.py +++ b/openhcs/interop/cellprofiler/runtime/function_contract_execution.py @@ -18,7 +18,6 @@ from openhcs.core.memory import detect_memory_type, stack_slices from openhcs.core.runtime_batch_contracts import ( Pure2DSliceBatchExecutor, - RuntimeBatchExecutionDomain, RuntimeBatchInvocationRequest, RuntimePure2DSliceBatchRequest, SliceIndexRuntimeParameter, @@ -76,21 +75,6 @@ def __init__( ) -> None: self.plane_projection = plane_projection - def invoke_raw( - self, - callable_contract: CallableContract, - func: Callable[..., RuntimeFunctionOutput], - image: RuntimeCallableArgument, - kwargs: RuntimeCallableKwargs, - ) -> RuntimeCallableArgument: - """Project the canonical ABI after the input domain has been selected.""" - return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.invocation( - func, - (callable_contract.raw_main_flow_call_argument(image),), - kwargs, - signature=callable_contract.raw_runtime_signature, - ).call() - def execute( self, callable_contract: CallableContract, @@ -232,14 +216,7 @@ def execute_pure_2d_slice_batch( if slice_request.slice_count <= 0: return [], 0.0 - declared_batch_executor = callable_contract.runtime_batch_executor( - RuntimeBatchExecutionDomain.PURE_2D_SLICES - ) - batch_executor = ( - declared_batch_executor - if callable(declared_batch_executor) - else Pure2DSliceBatchExecutor.default_executor() - ) + batch_executor = Pure2DSliceBatchExecutor.from_executors(callable_contract.runtime_batch_executors) slice_started_at = time.perf_counter() if batch_executor is not None and slice_request.slice_count > 1: slice_results = list(batch_executor(slice_request)) @@ -278,12 +255,12 @@ def execute_pure_3d( ), ) call_started_at = time.perf_counter() - result = self.invoke_raw( + result = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( callable_contract, func, projected_image, projected_kwargs, - ) + ).call() CellProfilerRuntimeProfileLogger.log_module_profile( "cp_full_stack_raw_call", time.perf_counter() - call_started_at, @@ -357,7 +334,7 @@ def execute_aligned_stack_slice( slice_index: int, slice_count: int, ) -> RuntimeCallableArgument: - return self.invoke_raw( + return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( callable_contract, slice_func, slice_payload, @@ -367,7 +344,7 @@ def execute_aligned_stack_slice( slice_count, reference_payload=slice_payload, ), - ) + ).call() slice_results, _slice_execute_seconds = self.execute_pure_2d_slice_batch( callable_contract, @@ -476,12 +453,12 @@ def execute_pure_2d( ) image_data = image_payload_data(image) if not isinstance(image_data, np.ndarray): - return self.invoke_raw( + return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( callable_contract, func, image, kwargs, - ) + ).call() prepare_started_at = time.perf_counter() memory_type = detect_memory_type(image_data) @@ -504,12 +481,12 @@ def execute_pure_2d( "declared plane projection. Kwargs cannot create image-axis " "execution semantics." ) - return self.invoke_raw( + return _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( callable_contract, func, image, kwargs, - ) + ).call() declared_plane_axis = image_payload_metadata(image).plane_axis if declared_plane_axis is not self.plane_projection.axis: raise RuntimeSliceProjectionDeclarationError( @@ -609,12 +586,12 @@ def execute_volumetric_to_slice( callable_contract: CallableContract, **kwargs: RuntimeCallableArgument, ) -> RuntimeCallableArgument: - result_2d = self.invoke_raw( + result_2d = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( callable_contract, func, image, kwargs, - ) + ).call() result_data = image_payload_data(result_2d) result_mask = image_payload_mask(result_2d) result_metadata = image_payload_metadata(result_2d) @@ -642,12 +619,12 @@ def execute_pure_2d_slice( sliced_kwargs = dict(sliced_kwargs) slice_index_name = SliceIndexRuntimeParameter.require_parameter_name() sliced_kwargs[slice_index_name] = slice_index - result = self.invoke_raw( + result = _CELLPROFILER_RUNTIME_CALLABLE_POLICY.contract_invocation( callable_contract, func, slice_2d, sliced_kwargs, - ) + ).call() return contextualize_main_image_output(slice_2d, result) def slice_pure_2d_kwargs( diff --git a/openhcs/processing/backends/lib_registry/registry_service.py b/openhcs/processing/backends/lib_registry/registry_service.py index 36459249c1..b730021e07 100644 --- a/openhcs/processing/backends/lib_registry/registry_service.py +++ b/openhcs/processing/backends/lib_registry/registry_service.py @@ -21,7 +21,7 @@ from zmqruntime import OperationCancellation from zmqruntime.client import endpoint_process -from openhcs.core.callable_contract import CallableContract, CallableProjection +from openhcs.core.callable_contract import CallableProjection from openhcs.core.function_reference import ResolvedRegistryFunction from openhcs.core.processing_preparation import ( CallablePreparation, @@ -163,7 +163,7 @@ def prepare_in_current_process( ) emit_status("Resolving prepared callable signatures") for func in callables: - CallableContract.warm_canonical_signature(func) + CallableProjection.from_callable(func).warm_canonical_signature() emit_status(f"Registered kernels ready ({len(callables)} callables)") emit_status(f"Function catalog ready ({len(cls._metadata_cache)} callables)") return cls._metadata_cache diff --git a/openhcs/processing/backends/lib_registry/unified_registry.py b/openhcs/processing/backends/lib_registry/unified_registry.py index f6097a5e74..a30e9af840 100644 --- a/openhcs/processing/backends/lib_registry/unified_registry.py +++ b/openhcs/processing/backends/lib_registry/unified_registry.py @@ -75,7 +75,6 @@ from openhcs.core.runtime_array_values import RuntimeArrayPayload, is_array_payload from openhcs.core.runtime_batch_contracts import ( Pure2DSliceBatchExecutor, - RuntimeBatchExecutionDomain, RuntimePure2DSliceBatchRequest, runtime_batch_executors_from_callable, ) @@ -283,6 +282,16 @@ class RuntimeCallablePolicy: RuntimeInvocationKwargPolicy.PASS_THROUGH ) + def contract_invocation( + self, contract: "CallableContract", func: Callable[..., Any], + image: Any, kwargs: Mapping[str, Any], + ) -> RuntimeCallableInvocation: + """Bind the prepared main-image carrier and actual raw execution ABI.""" + return self.invocation( + func, (contract.raw_main_flow_call_argument(image),), kwargs, + signature=contract.raw_runtime_signature, + ) + def invocation( self, func: Callable[..., Any], @@ -1813,14 +1822,7 @@ def execute_pure_2d(self, func, image, *args, **kwargs): kwargs.setdefault(parameter.name, value) args = args[len(positional_parameters) :] slices = slicer.slice_value(image, input_memory_type) - declared_batch_executor = runtime_batch_executors_from_callable(func).get( - RuntimeBatchExecutionDomain.PURE_2D_SLICES - ) - batch_executor = ( - declared_batch_executor - if callable(declared_batch_executor) - else Pure2DSliceBatchExecutor.default_executor() - ) + batch_executor = Pure2DSliceBatchExecutor.from_executors(runtime_batch_executors_from_callable(func)) def execute_slice( slice_func: Callable[..., Any], diff --git a/tests/unit/test_callable_abi_preparation.py b/tests/unit/test_callable_abi_preparation.py index 4e3e816fa5..8b050ad204 100644 --- a/tests/unit/test_callable_abi_preparation.py +++ b/tests/unit/test_callable_abi_preparation.py @@ -14,7 +14,7 @@ import openhcs.core.callable_contract as contract_module from openhcs.core.callable_contract import ( - CallableContract, CallableImportIdentity, CallableMetadata, + CallableContract, CallableImportIdentity, CallableMetadata, CallableProjection, attach_callable_contract_metadata, attach_processing_prepare, ) from openhcs.core.function_contract_metadata import FunctionContractAttribute @@ -23,7 +23,10 @@ from openhcs.core.processing_preparation import CallablePreparation, PreparationCacheBatch from openhcs.core.runtime_image_values import ImagePayloadMetadata, image_payload_data from openhcs.processing.backends.lib_registry.registry_service import RegistryService -from openhcs.core.runtime_batch_contracts import runtime_callable_defaults +from openhcs.core.runtime_batch_contracts import ( + Pure2DSliceBatchExecutor, RuntimeBatchExecutionDomain, + SerialPure2DSliceBatchExecutor, runtime_callable_defaults, +) from openhcs.core.pipeline.function_contracts import resolved_callable_parameter from openhcs.processing.backends.lib_registry.unified_registry import ( RuntimeCallablePolicy, RuntimeInvocationKwargPolicy, @@ -204,7 +207,7 @@ def raw(image, values=None): def test_supported_declaration_changes_invalidate_published_signature(mutation): def raw(image: np.ndarray): return image - CallableContract.warm_canonical_signature(raw) + CallableProjection.from_callable(raw).warm_canonical_signature() assert FunctionContractAttribute.canonical_signature in vars(raw) mutation(raw) assert FunctionContractAttribute.canonical_signature not in vars(raw) @@ -221,7 +224,7 @@ def raw(image): def test_exact_raw_default_filter_and_type_views_refresh_with_library_snapshot(): def raw(image: np.ndarray, scale: float = 2): return image * scale - CallableContract.warm_canonical_signature(raw) + CallableProjection.from_callable(raw).warm_canonical_signature() policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) assert runtime_callable_defaults(raw)["scale"] == 2 assert resolved_callable_parameter(raw, "scale").annotation is float @@ -230,7 +233,7 @@ def raw(image: np.ndarray, scale: float = 2): raw.__annotations__["scale"] = int assert runtime_callable_defaults(raw)["scale"] == 2 assert resolved_callable_parameter(raw, "scale").annotation is float - CallableContract.warm_canonical_signature(raw) + CallableProjection.from_callable(raw).warm_canonical_signature() assert runtime_callable_defaults(raw)["scale"] == 4 assert resolved_callable_parameter(raw, "scale").annotation is int @@ -241,7 +244,7 @@ def raw(image: np.ndarray, private: int = 8): def wrapper(image: np.ndarray, public: int = 2): return image * public wrapper.__dict__[FunctionContractAttribute.raw_processing_function] = raw - CallableContract.warm_canonical_signature(wrapper) + CallableProjection.from_callable(wrapper).warm_canonical_signature() assert runtime_callable_defaults(wrapper) == {"public":2} assert resolved_callable_parameter(wrapper, "public").annotation is int policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) @@ -257,7 +260,7 @@ def raw(image, labels): return image raw.__annotations__ = {"image":Annotated[np.ndarray,"image"],"labels":tuple[Annotated[int,"id"],...],"return":Annotated[np.ndarray,"result"]} expected = get_type_hints(raw) - CallableContract.warm_canonical_signature(raw) + CallableProjection.from_callable(raw).warm_canonical_signature() assert resolved_callable_parameter(raw,"image").annotation == expected["image"] assert resolved_callable_parameter(raw,"labels").annotation == expected["labels"] assert CallableContract.from_callable(raw).canonical_parameter_annotations["labels"] == raw.__annotations__["labels"] @@ -290,7 +293,7 @@ def wrapper(*args, **kwargs): assert second.raw_runtime_signature.parameters["scale"].annotation is float -def test_same_runtime_layout_reuses_canonical_without_comparing_mutable_defaults(fake_preparation): +def test_distinct_targets_own_signatures_even_when_layouts_match_without_comparing_defaults(fake_preparation): class RejectEquality: def __eq__(self, other): raise AssertionError("Signature reuse compared a mutable default") @@ -301,6 +304,31 @@ def raw(image: np.ndarray, values=default) -> np.ndarray: def wrapper(*args, **kwargs): return raw(*args, **kwargs) contract = CallableContract.from_prepared_callable(wrapper) + assert contract.metadata.raw_runtime_signature is not None + assert contract.raw_runtime_signature is not contract.canonical_signature + assert contract.raw_runtime_signature.parameters["values"].default is default + raw.__defaults__ = ("new raw default",) + second = CallableContract.from_prepared_callable(wrapper) + assert second.raw_runtime_signature.parameters["values"].default == "new raw default" + assert contract.raw_runtime_signature.parameters["values"].default is default + + +def test_same_actual_target_shares_canonical_signature(fake_preparation): + contract = CallableContract.from_prepared_callable(raw_numeric) assert contract.metadata.raw_runtime_signature is None assert contract.raw_runtime_signature is contract.canonical_signature - assert contract.raw_runtime_signature.parameters["values"].default is default + + +@pytest.mark.parametrize("executors", (None, {}, {RuntimeBatchExecutionDomain.PURE_2D_SLICES: None})) +def test_pure_2d_executor_selection_retains_serial_fallback(executors): + assert isinstance(Pure2DSliceBatchExecutor.from_executors(executors), SerialPure2DSliceBatchExecutor) + + +def test_pure_2d_executor_selection_retains_exact_declared_executor(): + def declared(*args, **kwargs): + raise AssertionError("Selecting an executor must not execute it") + + selected = Pure2DSliceBatchExecutor.from_executors({ + RuntimeBatchExecutionDomain.PURE_2D_SLICES: declared, + }) + assert selected is declared diff --git a/tests/unit/test_cellprofiler_function_contract_execution.py b/tests/unit/test_cellprofiler_function_contract_execution.py index 8ad2e4dbef..76ca8ec07f 100644 --- a/tests/unit/test_cellprofiler_function_contract_execution.py +++ b/tests/unit/test_cellprofiler_function_contract_execution.py @@ -27,6 +27,7 @@ from openhcs.core.callable_contract import ( CallableContract, CallableMetadata, + CallableProjection, callable_request, ) from openhcs.core.runtime_plane_projection import ( @@ -55,6 +56,9 @@ morphologicalskeleton, ) from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.processing.backends.lib_registry.unified_registry import ( + RuntimeCallablePolicy, RuntimeInvocationKwargPolicy, +) def _compiled_contract( @@ -422,9 +426,9 @@ def __init__(self, plane_projection=None): super().__init__(plane_projection=plane_projection) constructed.append(self) - def invoke_raw(self, callable_contract, func, image, kwargs): + def execute_pure_2d_slice(self, *args, **kwargs): raw_calls.append(self) - return super().invoke_raw(callable_contract, func, image, kwargs) + return super().execute_pure_2d_slice(*args, **kwargs) def raw(image: np.ndarray) -> np.ndarray: assert isinstance(image, np.ndarray) @@ -456,7 +460,8 @@ def foreign_raw(image: ArrayPayload) -> ArrayPayload: foreign_contract = _compiled_contract(foreign_raw, ProcessingContract.PURE_2D) child = constructed[1] - assert child.invoke_raw(foreign_contract, foreign_raw, source, {}) is source + policy = RuntimeCallablePolicy(kwarg_policy=RuntimeInvocationKwargPolicy.SIGNATURE_FILTERED) + assert policy.contract_invocation(foreign_contract, foreign_raw, source, {}).call() is source assert not hasattr(child, "_scope_contract") @@ -465,7 +470,7 @@ def test_actual_prepared_executor_planes_make_no_signature_or_hint_queries(monke def raw(image: np.ndarray, scale: int = 2) -> np.ndarray: calls.append(image.shape) return image * scale - CallableContract.warm_canonical_signature(raw) + CallableProjection.from_callable(raw).warm_canonical_signature() contract = _compiled_contract( raw, ProcessingContract.PURE_2D, artifact_outputs=(ArtifactSpec.output("Result",ImageArtifactType),), From 0b7f6f98a4b19d85d0dbb20f4db49e45cd0d90c1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 01:45:24 -0400 Subject: [PATCH 041/125] Validate explicit object planes without requiring acquisition provenance Refs #419. Follow up #429: independently authored runtime planes can own an exact object projection without acquisition source-plane records. Preserve source-axis and nonempty provenance cardinality checks, plus physical label/source shape validation. --- openhcs/core/runtime_object_label_building.py | 13 +++++---- .../test_saved_source_label_plane_domain.py | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/openhcs/core/runtime_object_label_building.py b/openhcs/core/runtime_object_label_building.py index cac8c54e0a..e7dba0d23b 100644 --- a/openhcs/core/runtime_object_label_building.py +++ b/openhcs/core/runtime_object_label_building.py @@ -142,13 +142,14 @@ def plane_semantics(self) -> RuntimePlaneAxisValueProjection | None: ) return None metadata = self.metadata - declared_projection = RuntimePlaneAxisValueProjection.from_source_declaration( - metadata.plane_axis, metadata.source_provenance, - ) - if declared_projection is not None: + if metadata.plane_axis is not None: + declared_count = metadata.source_provenance.source_plane_count if ( - projection.axis is not declared_projection.axis - or projection.axis_size != declared_projection.axis_size + projection.axis is not metadata.plane_axis + or ( + declared_count > 0 + and projection.axis_size != declared_count + ) ): raise ValueError( "Object-label plane projection conflicts with the source-image " diff --git a/tests/unit/test_saved_source_label_plane_domain.py b/tests/unit/test_saved_source_label_plane_domain.py index 2383b24259..a8c9f6c7ef 100644 --- a/tests/unit/test_saved_source_label_plane_domain.py +++ b/tests/unit/test_saved_source_label_plane_domain.py @@ -41,6 +41,34 @@ def _saved_label_input(source): ) +@pytest.mark.parametrize("axis", tuple(RuntimePlaneAxis)) +@pytest.mark.parametrize("plane_count", (1, 2)) +def test_explicit_object_projection_preserves_axis_without_acquisition_provenance( + axis, plane_count, +): + labels = np.ones((plane_count, 6, 7), dtype=np.int32) + source = ImagePayloadMetadata(plane_axis=axis).payload_with(labels) + projection = RuntimePlaneAxisValueProjection.preserve( + axis=axis, axis_size=plane_count, + ) + payload = SourceImageObjectLabelBuildRequest( + image=source, labels=labels, plane_projection=projection, + ).payload() + + assert payload.plane_axis is axis + assert payload.domain.declared_object_id_domains == ((1,),) * plane_count + assert payload.source_provenance == source.metadata.source_provenance + np.testing.assert_array_equal(payload.labels, labels) + other_axis = next(member for member in RuntimePlaneAxis if member is not axis) + with pytest.raises(ValueError, match="conflicts with the source-image axis"): + SourceImageObjectLabelBuildRequest( + image=source, labels=labels, + plane_projection=RuntimePlaneAxisValueProjection.preserve( + axis=other_axis, axis_size=plane_count, + ), + ).payload() + + @pytest.mark.parametrize("axis", tuple(RuntimePlaneAxis)) def test_original_artifact_admission_and_full_stack_shape_keep_2d_planes(axis): labels = np.zeros((2, 8, 9), dtype=np.int32) From 585413317f513b8b183326e5880504c8ac17700a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 01:52:59 -0400 Subject: [PATCH 042/125] Record prepared ABI owner gates and unresolved scientific qualification failures Refs #384, #419, #433. Preserve exact source-pinned controls, original architecture guard outcomes, confirmed OOM evidence and partial cohort capture limits. --- .../shared_runtime_plumbing_384_20261001.rst | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index c6181ca111..13157c1bda 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -592,3 +592,61 @@ Source freezes, complete observations and diagnostic boundaries are retained in ``/var/tmp/openhcs-owned-metadata-90c30-ordinary-abba-20261002/`` and ``/var/tmp/openhcs-owned-metadata-phase-probe-20261002/``. The original 026 comparison and original failures remain separate pinned evidence. + + +Prepared callable owner and independent plane follow-up (2026-10-02) +------------------------------------------------------------------ + +Registry preparation now resolves canonical and actual raw signatures after all +function preparation hooks and before READY. Authored declarations resolve in +compilation. Immutable signature pairs travel on the existing CallableMetadata +through existing function-reference/worker transport. Runtime filtering, batch +defaults and canonical argument admission consume the associated prepared view; +three separate signature/default/type LRU caches and late kernel preparation are +removed. Server preparation remains outside pipeline clocks. + +The isolated owner repair ``760e9ae8613d8737411a627b6abfe6a15f4ca0b9`` moves actual +target resolution, signature state and carrier grammar onto CallableMetadata; +CallableProjection/Reader own preparation admission, RuntimeCallablePolicy owns +invocation, and Pure2DSliceBatchExecutor owns the formerly duplicated executor +selection. Distinct actual targets always receive independent raw snapshots; +only the same actual target shares its canonical snapshot. No new wrapper class +or equality comparison over arbitrary callable defaults is introduced. + +Original, unchanged R0 passes against both the prepared-callable commit and the +frozen qualification source, eliminating its new class-size, Boolean-chain and +foreign-absence debt. Original R1 passes within the unchanged 160-second budget +with 3,084 projections. These are isolated gates; the inherited scalar-owner +per-file R0 gate on the whole branch remains open. Focused owner tests pass 105 +controls; the larger 648-pass/five-skip suite initially retained two baseline-red +plane-domain controls. Their failure also reproduces on untouched ``94915``. + +The independent repair ``0b7f6f98a4b19d85d0dbb20f4db49e45cd0d90c1`` validates an +explicit object projection against its declared source axis and nonempty +acquisition cardinality directly. Independently authored runtime planes can own +an exact projection without acquisition-plane provenance. Physical source/label +cardinality and spatial-shape checks remain mandatory. Thirty-nine controls pass, +including both axes at one/two planes and wrong-axis refusal. No object-domain +semantics are inferred from source storage axes. + +The all-30 scientific qualification on frozen ``94915`` is RED: 24 cases pass; +four fail at missing payload context in Align/Crop, one loses its server to a +confirmed kernel OOM kill, and one differs in threshold entropy. Saved input +replay traces that entropy mismatch to missing Threshold source scale/dtype at +the canonical raw boundary. Truthful declarations are being corrected without +scientific-body or tolerance changes. Linux killed endpoint PID 2412356 with +13,360,004 KiB anonymous RSS; the dominant allocation/retaining owner is still +unmeasured. Issue #433 tracks that distinct execution-server acceptance gate and +is formally linked to this draft PR. Retained native clocks are not current +paired speedup evidence. + +Evidence stays in the external benchmark RUNS directory, including +``pr394-94915-all30-native-science-qualification-red-20261002.json``, +``pr394-signature-owner-760-source-inventory-20261002.json``, the original R0/R1 +receipts, and ``issue419-explicit-object-plane-cardinality-followup-20261002.json``. +Two partial cohort captures complete all 31 production steps and match all six +CSV files and 120 TIFF dtype/shape/pixel arrays exactly, but each admits only six +of ten graph snapshots. V4 identifies an unsupported native allocation owner in +the four remaining before-state graphs; those graphs remain missing. Diagnostic +capture clocks do not establish a speedup, and the additional multi-second +whole-value plumbing payoff remains unmeasured. From 9ae4c9426cb767d3b2cbe997799bd0d211f16c4d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 01:54:23 -0400 Subject: [PATCH 043/125] Preserve image context for prepared CellProfiler calls (#419) --- .../backends/cellprofiler/alignment.py | 3 +- .../backends/cellprofiler/area_occupied.py | 2 +- .../processing/backends/cellprofiler/crop.py | 5 +- .../backends/cellprofiler/image_quality.py | 3 +- .../backends/cellprofiler/intensity.py | 2 +- .../backends/cellprofiler/manual_objects.py | 3 +- .../backends/cellprofiler/object_images.py | 3 +- .../backends/cellprofiler/thresholding.py | 5 +- .../processing/backends/cellprofiler/worms.py | 12 +- ...rofiler_additional_prepared_payload_abi.py | 184 ++++++++++++++++++ 10 files changed, 206 insertions(+), 16 deletions(-) create mode 100644 tests/unit/test_cellprofiler_additional_prepared_payload_abi.py diff --git a/openhcs/processing/backends/cellprofiler/alignment.py b/openhcs/processing/backends/cellprofiler/alignment.py index 9ef9b11ba2..b32e4a5ef5 100644 --- a/openhcs/processing/backends/cellprofiler/alignment.py +++ b/openhcs/processing/backends/cellprofiler/alignment.py @@ -40,6 +40,7 @@ RuntimePlaneAxisValueProjection, ) from openhcs.core.runtime_slice_projection import RuntimeSliceProjection +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, MaskedImagePayload, @@ -1083,7 +1084,7 @@ def prepare_align() -> None: @required_variable_components(VariableComponents.CHANNEL) @numpy(contract=ProcessingContract.PURE_3D) def align( - image: np.ndarray, + image: RuntimeArrayData, *, method: AlignModule.Method = AlignModule.Method.MUTUAL_INFORMATION, crop_mode: AlignModule.CropMode = AlignModule.CropMode.KEEP_SIZE, diff --git a/openhcs/processing/backends/cellprofiler/area_occupied.py b/openhcs/processing/backends/cellprofiler/area_occupied.py index 44049e52b8..e748871e96 100644 --- a/openhcs/processing/backends/cellprofiler/area_occupied.py +++ b/openhcs/processing/backends/cellprofiler/area_occupied.py @@ -728,7 +728,7 @@ def measure(self) -> AreaOccupiedMeasurement: SliceIndexRuntimeParameter, ) def measure_image_area_occupied( - image: np.ndarray, + image: RuntimeArrayData, *, operand_choices: Sequence[OperandChoice] = (OperandChoice.BINARY_IMAGE,), area_occupied_rows: Sequence[AreaOccupiedRow] = (), diff --git a/openhcs/processing/backends/cellprofiler/crop.py b/openhcs/processing/backends/cellprofiler/crop.py index 62fd7e1ba9..79db9fe1e5 100644 --- a/openhcs/processing/backends/cellprofiler/crop.py +++ b/openhcs/processing/backends/cellprofiler/crop.py @@ -24,6 +24,7 @@ ) from openhcs.core.pipeline.function_contracts import special_inputs from openhcs.core.runtime_measurements import RuntimeMeasurementFeature +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, @@ -608,7 +609,7 @@ def cropped_pixels(self) -> np.ndarray: class CropRequest: """Executable CellProfiler Crop request.""" - image: np.ndarray + image: RuntimeArrayData mask_plane: np.ndarray | None = None crop_shape: CropModule.Shape = CropModule.Shape.RECTANGLE cropping_method: CropModule.Method = CropModule.Method.COORDINATES @@ -857,7 +858,7 @@ def crop_output_metadata( @numpy_decorator(contract=ProcessingContract.PURE_2D) @special_inputs("topology_inputs") def crop( - image: np.ndarray, + image: RuntimeArrayData, topology_inputs: tuple[np.ndarray | ObjectLabelValue, ...] = (), crop_shape: CropModule.Shape = CropModule.Shape.RECTANGLE, cropping_method: CropModule.Method = CropModule.Method.COORDINATES, diff --git a/openhcs/processing/backends/cellprofiler/image_quality.py b/openhcs/processing/backends/cellprofiler/image_quality.py index e4338ba2f8..741d26e074 100644 --- a/openhcs/processing/backends/cellprofiler/image_quality.py +++ b/openhcs/processing/backends/cellprofiler/image_quality.py @@ -2,6 +2,7 @@ from __future__ import annotations +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.process_local_cache import RegisteredProcessLocalBoundedCache from abc import ABC, abstractmethod @@ -1565,7 +1566,7 @@ def _haralick_h3_numba(image: np.ndarray, scale: int) -> float: @numpy(contract=ProcessingContract.PURE_2D) def measure_image_quality( - image: np.ndarray, + image: RuntimeArrayData, include_scaling: bool = True, calculate_blur: bool = True, calculate_saturation: bool = True, diff --git a/openhcs/processing/backends/cellprofiler/intensity.py b/openhcs/processing/backends/cellprofiler/intensity.py index aae7b8583d..b400696f06 100644 --- a/openhcs/processing/backends/cellprofiler/intensity.py +++ b/openhcs/processing/backends/cellprofiler/intensity.py @@ -1560,7 +1560,7 @@ def _object_intensity_batch_key( @object_label_input_execution_mode(ObjectLabelInputExecutionMode.SLICE_ALIGNED) @runtime_bound_parameters(SliceIndexRuntimeParameter) def measure_object_intensity( - image: np.ndarray, + image: RuntimeArrayData, labels: ObjectLabelValue, object_intensity_backend_provider: BackendProviderInput = DEFAULT_CELLPROFILER_BACKEND_SELECTION, slice_index: int = OBJECT_INTENSITY_DEFAULT_SLICE_INDEX, diff --git a/openhcs/processing/backends/cellprofiler/manual_objects.py b/openhcs/processing/backends/cellprofiler/manual_objects.py index f4c64435b8..6ae87fa586 100644 --- a/openhcs/processing/backends/cellprofiler/manual_objects.py +++ b/openhcs/processing/backends/cellprofiler/manual_objects.py @@ -16,6 +16,7 @@ import numpy as np +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.artifacts import ( ArtifactSpec, ImageArtifactType, @@ -63,7 +64,7 @@ class ManualObjectStats: ) @numpy(contract=ProcessingContract.PURE_2D) def identify_objects_manually( - image: np.ndarray, + image: RuntimeArrayData, labels_input: np.ndarray | None = None, ) -> tuple[ np.ndarray, diff --git a/openhcs/processing/backends/cellprofiler/object_images.py b/openhcs/processing/backends/cellprofiler/object_images.py index 2ec98235ff..81912cc368 100644 --- a/openhcs/processing/backends/cellprofiler/object_images.py +++ b/openhcs/processing/backends/cellprofiler/object_images.py @@ -10,6 +10,7 @@ from metaclass_registry import AutoRegisterMeta import numpy as np +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.artifacts import ImageArtifactType, ObjectLabelsArtifactType from openhcs.core.memory import numpy as numpy_decorator from openhcs.core.measurement_row_materialization import ( @@ -181,7 +182,7 @@ def object_label_colormap(colormap_name: str, num_labels: int) -> np.ndarray: @numpy_decorator(contract=ProcessingContract.PURE_2D) def convert_image_to_objects( - image: np.ndarray, + image: RuntimeArrayData, cast_to_bool: bool = False, preserve_label: bool = False, background: int = 0, diff --git a/openhcs/processing/backends/cellprofiler/thresholding.py b/openhcs/processing/backends/cellprofiler/thresholding.py index eeac9dd3fb..7cd50a34d1 100644 --- a/openhcs/processing/backends/cellprofiler/thresholding.py +++ b/openhcs/processing/backends/cellprofiler/thresholding.py @@ -45,6 +45,7 @@ from openhcs.core.pipeline.function_contracts import runtime_bound_parameters from openhcs.core.public_api import public_names_from_objects from openhcs.core.registry_strategies import EnumKeyedStrategyMixin +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_intensity_scale_for_dtype, @@ -549,7 +550,7 @@ class GlobalThresholdSourceSelection: method_parameters: GlobalThresholdMethodParameters -def normalize_cellprofiler_image(image: np.ndarray) -> np.ndarray: +def normalize_cellprofiler_image(image: RuntimeArrayData) -> np.ndarray: """Return an image in CellProfiler's normalized pixel-data convention.""" return image_payload_data( normalize_cellprofiler_image_payload( @@ -2578,7 +2579,7 @@ class _ThresholdEmbeddedMaskRuntimeParameter(KeywordRuntimeParameter): @runtime_image_execution_mode(ImagePayloadExecutionMode.FULL_STACK) @numpy(contract=ProcessingContract.PURE_2D) def threshold( - image: np.ndarray, + image: RuntimeArrayData, mask: np.ndarray | None = None, threshold_scope: ThresholdScope = ThresholdScope.GLOBAL, threshold_method: ThresholdMethod = ThresholdMethod.OTSU, diff --git a/openhcs/processing/backends/cellprofiler/worms.py b/openhcs/processing/backends/cellprofiler/worms.py index 74359cf712..1f0964c1a6 100644 --- a/openhcs/processing/backends/cellprofiler/worms.py +++ b/openhcs/processing/backends/cellprofiler/worms.py @@ -1709,7 +1709,7 @@ def canonical_image_output( def _execute_untangle_worms( - image: np.ndarray, + image: RuntimeArrayData, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, num_control_points: int = 21, @@ -1868,7 +1868,7 @@ def _execute_untangle_worms( def _untangle_worms_output( - image: np.ndarray, + image: RuntimeArrayData, *, expected_style: OverlapStyle, overlap_style: OverlapStyle, @@ -1940,7 +1940,7 @@ def _untangle_worms_output( @numpy(contract=ProcessingContract.PURE_2D) def untangle_worms( - image: np.ndarray, + image: RuntimeArrayData, overlap_style: OverlapStyle = OverlapStyle.WITHOUT_OVERLAP, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, @@ -1993,7 +1993,7 @@ def untangle_worms( @numpy(contract=ProcessingContract.PURE_2D) def untangle_worms_with_overlap( - image: np.ndarray, + image: RuntimeArrayData, overlap_style: OverlapStyle = OverlapStyle.WITH_OVERLAP, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, @@ -2046,7 +2046,7 @@ def untangle_worms_with_overlap( @numpy(contract=ProcessingContract.PURE_2D) def untangle_worms_both( - image: np.ndarray, + image: RuntimeArrayData, overlap_style: OverlapStyle = OverlapStyle.BOTH, min_worm_area: float = 100.0, max_worm_area: float = 5000.0, @@ -2226,7 +2226,7 @@ def straighten_worms( @numpy(contract=ProcessingContract.PURE_2D) def identify_dead_worms( - image: np.ndarray, + image: RuntimeArrayData, worm_width: int = 10, worm_length: int = 100, angle_count: int = 32, diff --git a/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py b/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py new file mode 100644 index 0000000000..39fa425644 --- /dev/null +++ b/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py @@ -0,0 +1,184 @@ +"""Prepared raw arguments retain context consumed by nominal CP request owners.""" +from dataclasses import replace + +import numpy as np +import pytest + +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.callable_contract import CallableContract +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, + normalize_image_payload_intensity, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter +from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( + CellProfilerFunctionContractExecutor, +) +from openhcs.processing.backends.cellprofiler.alignment import AlignModule, align +from openhcs.processing.backends.cellprofiler.area_occupied import measure_image_area_occupied +from openhcs.processing.backends.cellprofiler.crop import CropModule, crop +from openhcs.processing.backends.cellprofiler.image_quality import measure_image_quality +from openhcs.processing.backends.cellprofiler.intensity import measure_object_intensity +from openhcs.processing.backends.cellprofiler.manual_objects import identify_objects_manually +from openhcs.processing.backends.cellprofiler.object_images import convert_image_to_objects +from openhcs.processing.backends.cellprofiler.thresholding import threshold +from openhcs.processing.backends.cellprofiler.worms import ( + identify_dead_worms, + untangle_worms, + untangle_worms_both, + untangle_worms_with_overlap, +) + + +def _prepared_contract(func): + contract = CallableContract.from_callable(func).with_prepared_signature() + return replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + + +@pytest.mark.parametrize( + "func", + ( + align, + crop, + measure_image_area_occupied, + measure_object_intensity, + measure_image_quality, + identify_objects_manually, + convert_image_to_objects, + identify_dead_worms, + threshold, + untangle_worms, + untangle_worms_with_overlap, + untangle_worms_both, + ), +) +def test_prepared_primary_context_consumers_keep_nominal_payload_and_bare_pixels(func): + pixels = np.arange(60, dtype=np.float32).reshape(4, 5, 3) + mask = np.ones((4, 5), dtype=bool) + mask[0] = False + source = ImagePayloadMetadata( + source_path="/input/rgb.tif", + source_image_names=("RGB",), + source_channel_axis=-1, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(8, 10) + ), + ).payload_with(pixels, mask) + contract = _prepared_contract(func) + + projected = contract.raw_main_flow_call_argument(source) + + assert projected is source + assert image_payload_mask(projected) is mask + assert image_payload_metadata(projected).source_spatial_domain.origin_yx == (2, 3) + assert image_payload_metadata(projected).source_image_paths == ("/input/rgb.tif",) + assert contract.raw_main_flow_call_argument(pixels) is pixels + + +@pytest.mark.parametrize("color", (False, True)) +def test_prepared_align_preserves_named_plane_axis_masks_and_source_identity(color): + shape = (8, 9, 3) if color else (8, 9) + image = np.random.default_rng(1).random(shape, dtype=np.float32) + pixels = np.stack((image, image)) + masks = np.ones((2, 8, 9), dtype=bool) + masks[:, 0] = False + metadata = ImagePayloadMetadata( + source_image_names=("First", "Second"), + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/input/first.tif", "/input/second.tif"), + component_metadata=({"well": "A01", "channel": "1"}, {"well": "A01", "channel": "2"}), + ), + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_channel_axis=3 if color else None, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(14, 17) + ), + ) + source = metadata.payload_with(pixels, masks) + contract = _prepared_contract(align) + + outputs, measurements = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + {"method": AlignModule.Method.NORMALIZED_CROSS_CORRELATION, + "crop_mode": AlignModule.CropMode.KEEP_SIZE}, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + assert len(outputs.slices) == 2 + assert measurements.row_count() == 2 + for index, output in enumerate(outputs.slices): + np.testing.assert_array_equal(image_payload_data(output), pixels[index]) + np.testing.assert_array_equal(image_payload_mask(output), masks[index]) + output_metadata = image_payload_metadata(output) + assert output_metadata.source_image_paths == (("/input/first.tif", "/input/second.tif")[index],) + assert output_metadata.source_spatial_domain == metadata.source_spatial_domain + assert output_metadata.source_channel_axis == (2 if color else None) + + +def test_prepared_crop_preserves_rgb_spatial_mask_and_calibrated_source_domain(): + pixels = np.arange(60, dtype=np.float32).reshape(4, 5, 3) + parent_mask = np.ones((4, 5), dtype=bool) + parent_mask[0] = False + domain = SourceSpatialDomain(origin_yx=(2, 3), source_shape_yx=(8, 10)) + source = ImagePayloadMetadata( + source_channel_axis=-1, + source_image_names=("RGB",), + source_path="/input/rgb.tif", + source_spatial_domain=domain, + ).payload_with(pixels, parent_mask) + contract = _prepared_contract(crop) + + output, cropping, measurements = CellProfilerFunctionContractExecutor().execute( + contract, + contract.resolve_canonical_raw_callable(), + source, + {"left_right_rectangle_positions": (1, 4), + "top_bottom_rectangle_positions": (0, 3), + "removal_method": CropModule.RemovalMethod.NO}, + execution_mode=ImagePayloadExecutionMode.NATURAL, + ) + + expected_crop = np.zeros((4, 5), dtype=bool) + expected_crop[:3, 1:4] = True + expected_pixels = pixels.copy() + expected_pixels[~expected_crop] = 0 + np.testing.assert_array_equal(cropping, expected_crop) + np.testing.assert_array_equal(image_payload_data(output), expected_pixels) + np.testing.assert_array_equal(image_payload_mask(output), expected_crop & parent_mask) + output_metadata = image_payload_metadata(output) + assert output_metadata.source_channel_axis == -1 + assert output_metadata.source_spatial_domain == domain + assert output_metadata.source_image_paths == ("/input/rgb.tif",) + assert measurements.row_count() == 1 + + +def test_prepared_threshold_keeps_owned_unit_interval_proof_and_mask(): + pixels = np.arange(20, dtype=np.uint8).reshape(4, 5) + mask = np.ones(pixels.shape, dtype=bool) + mask[0] = False + source = normalize_image_payload_intensity( + ImagePayloadMetadata.for_array(pixels).payload_with(pixels, mask), + dtype=np.float32, + ) + contract = _prepared_contract(threshold) + + projected = contract.raw_main_flow_call_argument(source) + + assert projected is source + assert image_payload_metadata(projected).unit_interval_intensity_scale == 255 + assert image_payload_mask(projected) is mask + np.testing.assert_array_equal(image_payload_data(projected), pixels.astype(np.float32) / 255) From 640cffa1c64e71e118f6a077549c98b45940049f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 02:01:09 -0400 Subject: [PATCH 044/125] Let the explicit plane projection validate its source declaration Refs #419. Keep axis/cardinality validation on the actual nominal proof owner, eliminating foreign metadata absence checks in the object builder. Existing independent-plane and acquisition-provenance controls pass unchanged. --- openhcs/core/runtime_object_label_building.py | 16 +++------------- openhcs/core/runtime_plane_projection.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/openhcs/core/runtime_object_label_building.py b/openhcs/core/runtime_object_label_building.py index e7dba0d23b..e47b3b06a2 100644 --- a/openhcs/core/runtime_object_label_building.py +++ b/openhcs/core/runtime_object_label_building.py @@ -142,19 +142,9 @@ def plane_semantics(self) -> RuntimePlaneAxisValueProjection | None: ) return None metadata = self.metadata - if metadata.plane_axis is not None: - declared_count = metadata.source_provenance.source_plane_count - if ( - projection.axis is not metadata.plane_axis - or ( - declared_count > 0 - and projection.axis_size != declared_count - ) - ): - raise ValueError( - "Object-label plane projection conflicts with the source-image " - "axis declaration." - ) + projection.validate_source_declaration( + metadata.plane_axis, metadata.source_provenance, + ) label_array = np.asarray(self.labels) projection.validate_shape( label_array.shape, diff --git a/openhcs/core/runtime_plane_projection.py b/openhcs/core/runtime_plane_projection.py index fe1bf18bc0..9f7affad17 100644 --- a/openhcs/core/runtime_plane_projection.py +++ b/openhcs/core/runtime_plane_projection.py @@ -297,6 +297,21 @@ def from_projector( axis_size=axis_size, ) + def validate_source_declaration( + self, axis: RuntimePlaneAxis | None, source: "SourceImageProvenance", + ) -> None: + """Check an explicit projection against retained acquisition declarations.""" + if axis is None: + return + declared_count = source.source_plane_count + if self.axis is not axis or ( + declared_count > 0 and self.axis_size != declared_count + ): + raise ValueError( + "Object-label plane projection conflicts with the source-image " + "axis declaration." + ) + @classmethod def require_from_projector( cls, From 7241ca99b4f00e48f2a742c8a1fdf57c2bd896db Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 02:03:27 -0400 Subject: [PATCH 045/125] Declare the intended object domain in the retained-plane regression control Refs #419, #427. The existing main test implicitly produced a PAYLOAD object domain then expected 2D plane measurements. Declare its exact singleton object-plane projection; keep source storage axes independent from global object IDs and genuine volumes. --- tests/unit/test_rescale_stream_plane_declaration.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_rescale_stream_plane_declaration.py b/tests/unit/test_rescale_stream_plane_declaration.py index 50e2cc6c1b..981d2d3c79 100644 --- a/tests/unit/test_rescale_stream_plane_declaration.py +++ b/tests/unit/test_rescale_stream_plane_declaration.py @@ -16,7 +16,7 @@ from openhcs.core.component_group_scope import RuntimeExecutionAxisScope from openhcs.core.function_patterns import InvocationArtifactInputEdgePlan from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest -from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis, RuntimePlaneAxisValueProjection from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.core.config import StepSourceBindingsConfig from openhcs.core.source_bindings import NamedSourceBinding @@ -162,7 +162,16 @@ def test_declared_source_plane_has_area_not_volume(retained_site): plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, ) payload = metadata.payload_with(pixels, None) - objects = SourceImageObjectLabelBuildRequest(image=payload, labels=labels).payload() + objects = SourceImageObjectLabelBuildRequest( + image=payload, + labels=labels, + plane_projection=( + RuntimePlaneAxisValueProjection.preserve( + axis=RuntimePlaneAxis.RUNTIME_SLICE, axis_size=1, + ) + if retained_site else None + ), + ).payload() _, rows = measure_object_size_shape.__wrapped__( payload, objects, calculate_advanced=False, calculate_zernikes=False, ) From 84e99c0ddefdc3adf656246254484c36b42ededd Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 02:19:15 -0400 Subject: [PATCH 046/125] Preserve source image context in prepared grid object calls (#419) --- .../processing/backends/cellprofiler/grid.py | 7 +-- ...rofiler_additional_prepared_payload_abi.py | 50 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/openhcs/processing/backends/cellprofiler/grid.py b/openhcs/processing/backends/cellprofiler/grid.py index ad92c9b044..3c83012bde 100644 --- a/openhcs/processing/backends/cellprofiler/grid.py +++ b/openhcs/processing/backends/cellprofiler/grid.py @@ -18,6 +18,7 @@ from openhcs.constants.constants import VariableComponents from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.artifacts import ( ArtifactSpecCollection, ArtifactSpecRelation, @@ -1293,14 +1294,14 @@ class IdentifyObjectsInGridRequest(GridShapeContext): """Executable request for CellProfiler IdentifyObjectsInGrid semantics.""" registry_key = "identify_objects" - image: np.ndarray + image: RuntimeArrayData shape_choice: ShapeChoice @classmethod def from_runtime( cls, *, - image: np.ndarray, + image: RuntimeArrayData, grid_definition: GridRuntimeDefinitionRequest, shape_choice: ShapeChoice, diameter_choice: DiameterChoice, @@ -1485,7 +1486,7 @@ def draw_grid_overlay( @numpy(contract=ProcessingContract.PURE_2D) @special_inputs("topology_inputs") def identify_objects_in_grid( - image: np.ndarray, + image: RuntimeArrayData, topology_inputs: tuple[SpatialGrid | ObjectLabelValue, ...], grid_rows: int = 8, grid_columns: int = 12, diff --git a/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py b/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py index 39fa425644..1ceb91da58 100644 --- a/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py +++ b/tests/unit/test_cellprofiler_additional_prepared_payload_abi.py @@ -23,6 +23,10 @@ from openhcs.processing.backends.cellprofiler.alignment import AlignModule, align from openhcs.processing.backends.cellprofiler.area_occupied import measure_image_area_occupied from openhcs.processing.backends.cellprofiler.crop import CropModule, crop +from openhcs.processing.backends.cellprofiler.grid import ShapeChoice, identify_objects_in_grid +from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest +from openhcs.core.runtime_spatial_grid import SpatialGrid +from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.processing.backends.cellprofiler.image_quality import measure_image_quality from openhcs.processing.backends.cellprofiler.intensity import measure_object_intensity from openhcs.processing.backends.cellprofiler.manual_objects import identify_objects_manually @@ -56,6 +60,7 @@ def _prepared_contract(func): measure_object_intensity, measure_image_quality, identify_objects_manually, + identify_objects_in_grid, convert_image_to_objects, identify_dead_worms, threshold, @@ -87,6 +92,51 @@ def test_prepared_primary_context_consumers_keep_nominal_payload_and_bare_pixels assert contract.raw_main_flow_call_argument(pixels) is pixels +@pytest.mark.parametrize("shape", ((5, 5), (4, 7))) +@pytest.mark.parametrize("shape_choice", (ShapeChoice.RECTANGLE, ShapeChoice.NATURAL, ShapeChoice.CIRCLE_NATURAL)) +def test_prepared_grid_objects_preserve_rgb_mask_calibration_and_crop_domain(shape, shape_choice): + pixels = np.arange(np.prod(shape) * 3, dtype=np.float32).reshape(*shape, 3) + mask = np.ones(shape, dtype=bool) + mask[0] = False + metadata = ImagePayloadMetadata( + source_path="/input/grid_rgb.tif", + source_channel_axis=-1, + source_voxel_spacing=SourceVoxelSpacing((2.0, 0.7, 0.9)), + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(shape[0] + 4, shape[1] + 6), + ), + ) + source = metadata.payload_with(pixels, mask) + grid = SpatialGrid( + name="Grid", rows=1, columns=1, x_spacing=4.0, y_spacing=4.0, + x_origin=2.0, y_origin=2.0, source_spatial_shape_yx=shape, + ) + guide_pixels = np.zeros(shape, dtype=np.int32) + guide_pixels[1:3, 1:4] = 1 + guides = SourceImageObjectLabelBuildRequest(image=source, labels=guide_pixels).payload() + topology_inputs = (grid,) if shape_choice is ShapeChoice.RECTANGLE else (grid, guides) + contract = _prepared_contract(identify_objects_in_grid) + + returned_image, _rows, objects = CellProfilerFunctionContractExecutor().execute( + contract, contract.resolve_canonical_raw_callable(), source, + {"topology_inputs": topology_inputs, "shape_choice": shape_choice}, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + assert returned_image is source + assert image_payload_mask(returned_image) is mask + assert objects.source_provenance == metadata.source_provenance + assert objects.source_spatial_domain.origin_yx == (2, 3) + assert objects.source_spatial_domain.source_shape_yx == metadata.source_spatial_domain.source_shape_yx + assert objects.parent_image_source_voxel_spacing == metadata.source_voxel_spacing + assert objects.labels.shape == shape + assert objects.domain.declared_object_count == 1 + assert objects.domain.declared_object_ids == (1,) + expanded = objects.source_spatial_domain.materialize(objects.labels, spatial_axes_yx=(0, 1)) + assert expanded.shape == metadata.source_spatial_domain.source_shape_yx + np.testing.assert_array_equal(expanded[2:2 + shape[0], 3:3 + shape[1]], objects.labels) + + @pytest.mark.parametrize("color", (False, True)) def test_prepared_align_preserves_named_plane_axis_masks_and_source_identity(color): shape = (8, 9, 3) if color else (8, 9) From 38067a4a5c348e0c7fcedd1d48f69bb99b7be19b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 02:24:01 -0400 Subject: [PATCH 047/125] Record prepared payload readiness and native parity qualification (#419) --- .../shared_runtime_plumbing_384_20261001.rst | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 13157c1bda..dcfe8e9257 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -650,3 +650,53 @@ of ten graph snapshots. V4 identifies an unsupported native allocation owner in the four remaining before-state graphs; those graphs remain missing. Diagnostic capture clocks do not establish a speedup, and the additional multi-second whole-value plumbing payoff remains unmeasured. + + +Prepared payload contract qualification (2026-10-02) +-------------------------------------------------- + +The integrated library/kernel readiness run at ``1ad3898`` prepares all 267 +functions and 976 targets before READY. It creates 265 distinct actual raw +signature snapshots and reuses the canonical snapshot for two identical actual +targets. Three rounds of prepared annotation, carrier, raw-signature, batch +default and invocation-construction queries perform zero additional live +signature/type-hint resolutions. The fresh-cache 90.381-second preparation is +server startup diagnostics, excluded from pipeline clocks. Later explicit-plane +validation and test-producer changes do not change these warming/signature owner +sources. This finite check does not claim zero reflection for arbitrary +unprepared authoring or unsupported callable mutation. + +The annotation-only follow-up ``9ae4c9426`` corrects twelve existing context +consumers, three forwarding helpers and CropRequest.image. Existing +RuntimeArrayData preserves their nominal image payload at the actual raw +boundary; NumPy-only consumers still receive NumPy arrays. Scientific bodies, +defaults, decorators and return declarations are unchanged by AST comparison. +Frozen ``7241ca99b`` passes native-science comparison for Colocalization, +Neighbors, YeastColonies and pixel classification with zero differences. +YeastPatches passes its previous Crop failure but subsequently exposes an +incorrect RGB source domain from IdentifyObjectsInGrid. + +Saved actual RGB pixels, source metadata and both 93-ID label artifacts isolate +the remaining defect: IdentifyObjectsInGrid's primary argument and its existing +request field/from_runtime argument falsely declare np.ndarray despite passing +the image to SourceImageObjectLabelBuildRequest. Their raw ABI strips the +1200-by-1600 source domain and (150, 170) crop origin, producing a (1255, 3) +domain for (835, 1255) labels. ``84e99c0dd`` changes only those declarations and +their direct RuntimeArrayData import. Scientific bodies and the refusing domain +guard are unchanged. Seventy-seven scoped controls and 127 integrated controls +pass. A fresh YeastPatches execution at that revision passes the existing native +comparator with zero differences. Its replay uses retained source-pinned native +artifacts, not newly timed native execution. The original 84-entry/38-module AST +frontier now has no remaining exact classmethod context leads; aliases, inherited +dispatch and dynamic routes remain outside that finite inventory. + +The five affected cases have passed across these two targeted sources. The full +all-30 latest-head science gate, the execution-server OOM retaining-owner gate, +the whole-branch scalar per-file architecture gate and fresh ordinary performance +qualification remain open. No current native speedup is inferred from retained +clocks or instrumented captures. Evidence includes +``issue419-actual-integrated-1ad-signature-owner-readiness-20261002.json``, +``pr394-7241-context5-science-qualification-20261002.json``, +``issue419-grid-context-fix-20261002.json`` and +``pr394-84e99-grid-science-qualification-20261002.json`` in the benchmark RUNS +directory. All original failed source-pinned runs remain retained. From 45f3a721e115b37ab662f362fa5ab4d5e58e1f6e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 02:28:08 -0400 Subject: [PATCH 048/125] Bind graph ROI source metadata through existing archive owners (#134) --- .../shared_runtime_plumbing_384_20261001.rst | 28 ++ openhcs/processing/materialization/core.py | 10 +- .../test_graph_roi_source_roundtrip_134.py | 272 ++++++++++++++++++ .../test_spatial_graph_materialization.py | 3 +- 4 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_graph_roi_source_roundtrip_134.py diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index dcfe8e9257..023b75894e 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -700,3 +700,31 @@ clocks or instrumented captures. Evidence includes ``issue419-grid-context-fix-20261002.json`` and ``pr394-84e99-grid-science-qualification-20261002.json`` in the benchmark RUNS directory. All original failed source-pinned runs remain retained. + +Graph ROI source-bearing writer integration (issue #134) +------------------------------------------------------- + +The receiving owner requested narrow integration from PR #404 checkpoint +``f3e566485a391a270a6d2c40dbfddf07881dcd3d``. The exact patch binds existing +ROIArchiveSourceMetadata to graph ROI content and places the same existing +ImagePayloadMetadata on Output. Graph provenance already selects its declared +source plane; no second global-plane selection is performed. Coordinate spacing +uses existing SourceVoxelSpacing. The binder, disk ZIP transport, geometry +projection and native admission retain their existing owners and guards. + +On current ``38067a4a5`` before the production patch, the receiving seven-case +fixture reproduces three source-bearing failures and four refusal controls +pass. After integration, all seven pass alongside the original graph, batch +outcome and materialization controls: 87 PASS in 2.28 seconds. The existing +geometry test obtains geometry through the original archive projection before +its unchanged coordinate and feature assertions; bound transport metadata is not +mistaken for a graph feature. An independent cooperative graph subtype and new +feature require no new writer or consumer registration. Missing, unbound, mixed +and conflicting native source declarations remain refused before transport. + +This is current-source synthetic writer/disk/archive qualification. Actual +installed public/native graph reopening remains with the receiving owner, and +no performance or biological acceptance claim follows. Original failures remain +in ``pr394-38067-graph-roi-before-20261002.log``; integrated controls are in +``pr394-graph-roi-integrated-after-20261002.log`` in RUNS. The historical PR #404 +branch and its older publication proposal are not merged as a whole. diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index a095267a20..61340a6503 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -73,6 +73,7 @@ source_component_metadata_value, source_metadata_value, ) +from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.source_spatial_domain import SourceSpatialDomain from openhcs.core.steps.stream_component_semantics import ( StreamImagePayloadMetadataProjector, @@ -3284,6 +3285,8 @@ def _write_spatial_graph_roi_zip( from polystore.roi import ROI, PolylineShape + from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata + materialization_input = MaterializationInput.from_value(data, options) graph = materialization_input.data if not isinstance(graph, SpatialGraph): @@ -3292,6 +3295,10 @@ def _write_spatial_graph_roi_zip( f"got {type(graph).__name__}." ) + source_metadata = ImagePayloadMetadata( + source_provenance=graph.source_provenance, + source_voxel_spacing=SourceVoxelSpacing(graph.coordinate_spacing), + ) rois = [] for edge in graph.edges: coordinates = np.asarray(edge.coordinates, dtype=float) @@ -3333,7 +3340,8 @@ def _write_spatial_graph_roi_zip( return [ Output( path=ctx.paths(options).primary_output_path(options), - content=rois, + content=ROIArchiveSourceMetadata.bind(rois, source_metadata), + metadata=source_metadata, ) ] diff --git a/tests/unit/test_graph_roi_source_roundtrip_134.py b/tests/unit/test_graph_roi_source_roundtrip_134.py new file mode 100644 index 0000000000..83f0b9b478 --- /dev/null +++ b/tests/unit/test_graph_roi_source_roundtrip_134.py @@ -0,0 +1,272 @@ +"""Receiving #134: actual graph context/writer/ZIP, without a native process.""" + +from pathlib import Path + +import numpy as np +import pytest +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import load_rois_from_zip + +from openhcs.core.artifacts import ( + ArtifactOutputPlan, + ArtifactSpec, + ObjectArtifactMemberSubjectRelation, + ObjectArtifactSubjectBinding, + ObjectLabelsArtifactType, + SpatialGraphArtifactType, +) +from openhcs.core.config import NapariStreamingConfig +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.runtime_spatial_graph import ( + SpatialGraph, + SpatialGraphEdge, + SpatialGraphNode, +) +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.steps.function_runtime import FunctionOutputContextStrategy +from openhcs.core.viewer_streaming_service import RoiStreamingRequest, StreamingService +from openhcs.processing.materialization import ( + MaterializationSpec, + SpatialGraphROIOptions, + materialization_outputs, + materialize, +) + + +@pytest.fixture(autouse=True) +def no_optional_storage_bootstrap(monkeypatch): + import polystore.base + + monkeypatch.setattr( + polystore.base, + "ensure_storage_registry", + lambda: pytest.fail("Provider/bootstrap execution is forbidden in this test"), + ) + + +class ProjectionAudit: + """Independent capability exercises cooperative construction and projection.""" + + def __init__(self, **kwargs): + self.projection_events = [] + super().__init__(**kwargs) + + def contextualized_source_provenance(self, provenance): + self.projection_events.append("before") + result = super().contextualized_source_provenance(provenance) + self.projection_events.append("after") + return result + + +class AuditedGraph(ProjectionAudit, SpatialGraph): + """New declaration; no writer, context strategy or reader registration.""" + + +def graph_and_plan(*, plane, graph_type=SpatialGraph, extra_features=None): + nodes = tuple( + SpatialGraphNode(index + 1, coordinate) + for index, coordinate in enumerate(((2.25, 3.5), (5.25, 9.5), (7.25, 12.5))) + ) + edges = tuple( + SpatialGraphEdge.from_features( + edge_id=index + 1, + source=nodes[index], + target=nodes[index + 1], + coordinates=np.asarray( + (nodes[index].coordinates, nodes[index + 1].coordinates) + ), + features={ + "neuron_label": 7, + "branch_distance_um": 8.5, + **(extra_features or {}), + }, + ) + for index in range(2) + ) + graph = graph_type( + name="declared_graph", + nodes=nodes, + edges=edges, + coordinate_spacing=(1.3556, 1.3556), + source_plane_index=plane, + ) + neurons = ArtifactSpec.output("neurons", ObjectLabelsArtifactType) + plan = ArtifactOutputPlan( + name=graph.name, + path="/engineering/graph.pkl", + artifact_type=SpatialGraphArtifactType, + relations=( + ObjectArtifactMemberSubjectRelation( + source=neurons.ref(), member_id_field="neuron_label" + ), + ), + producer_step_index=4, + producer_step_scope_id="engineering-step", + ) + return graph, plan + + +def contextualize(graph, plan): + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/engineering/nuclear.tif", "/engineering/body.tif"), + component_metadata=tuple( + { + "well": "A01", + "site": 1, + "channel": channel, + "z_index": 1, + "timepoint": 1, + } + for channel in (1, 2) + ), + ), + source_image_names=("nuclear", "body"), + source_voxel_spacing=SourceVoxelSpacing(graph.coordinate_spacing), + ) + source = metadata.payload_with(np.zeros((2, 16, 16), dtype=np.uint8)) + result = FunctionOutputContextStrategy.for_output_plan(plan).contextualize( + source, graph, plan, None + ) + assert result.source_provenance == metadata.source_provenance.for_source_plane( + graph.source_plane_index + ) + return result + + +def save_graph(tmp_path, graph, plan): + manager = FileManager({"disk": DiskStorageBackend()}) + path = materialize( + MaterializationSpec(SpatialGraphROIOptions()), + data=graph, + path=str(tmp_path / "misleading_w99_z999.roi.zip"), + filemanager=manager, + backends=["disk"], + backend_kwargs={}, + output_plan=plan, + ) + return Path(path), load_rois_from_zip(Path(path)) + + +@pytest.mark.parametrize("plane", [0, 1]) +def test_declared_source_plane_survives_graph_writer_zip_and_subject_features( + tmp_path, plane +): + graph, plan = graph_and_plan(plane=plane) + graph = contextualize(graph, plan) + (output,) = materialization_outputs( + MaterializationSpec(SpatialGraphROIOptions()), + data=graph, + path=str(tmp_path / "graph.roi.zip"), + filemanager=FileManager({"disk": DiskStorageBackend()}), + output_plan=plan, + ) + path, restored = save_graph(tmp_path, graph, plan) + metadata = ROIArchiveSourceMetadata.decode(restored) + assert metadata is not None, "Native graph ZIP lost its existing source declaration" + assert output.metadata.source_provenance == graph.source_provenance + assert metadata.source_provenance == graph.source_provenance + assert ( + metadata.source_path + == ("/engineering/nuclear.tif", "/engineering/body.tif")[plane] + ) + assert metadata.source_component_metadata["channel"] == plane + 1 + assert metadata.source_voxel_spacing == SourceVoxelSpacing(graph.coordinate_spacing) + assert metadata.plane_axis is None + assert metadata.source_spatial_domain == output.metadata.source_spatial_domain + assert path.name == "misleading_w99_z999.graph.roi.zip" + geometry = ROIArchiveSourceMetadata.geometry(restored) + assert len(geometry) == len(graph.edges) + for roi, edge in zip(geometry, graph.edges, strict=True): + np.testing.assert_array_equal(roi.shapes[0].coordinates, edge.coordinates) + assert roi.metadata["edge_id"] == edge.edge_id + assert roi.metadata["source_node_id"] == edge.source_node_id + assert roi.metadata["target_node_id"] == edge.target_node_id + assert roi.metadata["neuron_label"] == 7 + assert roi.metadata["branch_distance_um"] == 8.5 + assert roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_ID_FEATURE] == 7 + assert ( + '"object_labels","neurons","engineering-step",4' + in roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_FEATURE] + ) + assert ROIArchiveSourceMetadata.FIELD not in roi.metadata + assert ( + len( + { + roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_FEATURE] + for roi in geometry + } + ) + == 1 + ) + + +def test_independent_graph_capability_and_new_feature_need_no_consumers(tmp_path): + original, plan = graph_and_plan( + plane=1, + graph_type=AuditedGraph, + extra_features={"engineering_confidence": 0.375}, + ) + graph = contextualize(original, plan) + assert original.projection_events == ["before", "after"] + assert ( + graph.projection_events == [] + ) # cooperative constructor for contextualized copy + _, restored = save_graph(tmp_path, graph, plan) + metadata = ROIArchiveSourceMetadata.decode(restored) + assert metadata is not None, "New graph declaration lost its source declaration" + assert metadata.source_provenance == graph.source_provenance + assert all(roi.metadata["engineering_confidence"] == 0.375 for roi in restored) + assert all( + ROIArchiveSourceMetadata.FIELD not in roi.metadata + for roi in ROIArchiveSourceMetadata.geometry(restored) + ) + + +@pytest.mark.parametrize("invalid", ["absent", "unbound", "mixed", "conflicting"]) +def test_original_native_admission_refuses_bad_graph_sources_before_transport( + tmp_path, invalid +): + graph, plan = graph_and_plan(plane=1) + graph = contextualize(graph, plan) + path, rois = save_graph(tmp_path, graph, plan) + geometry = ROIArchiveSourceMetadata.geometry(rois) + metadata = ImagePayloadMetadata(source_provenance=graph.source_provenance) + bad_archives = { + "absent": geometry, + "unbound": ROIArchiveSourceMetadata.bind(geometry, ImagePayloadMetadata()), + "mixed": [ + ROIArchiveSourceMetadata.bind(geometry[:1], metadata)[0], + geometry[1], + ], + "conflicting": [ + ROIArchiveSourceMetadata.bind(geometry[:1], metadata)[0], + ROIArchiveSourceMetadata.bind( + geometry[1:], + metadata.replace_fields(source_path="/engineering/other.tif"), + )[0], + ], + } + DiskStorageBackend().save(bad_archives[invalid], path) + # No viewer/handler object: original source admission must reject before using either. + service = StreamingService( + FileManager({"disk": DiskStorageBackend()}), None, tmp_path + ) + with pytest.raises( + ValueError, match="Native ROI source metadata|missing or conflicting" + ): + service.stream_rois( + RoiStreamingRequest( + viewer=None, + config=NapariStreamingConfig(enabled=True), + status_callback=lambda _: None, + error_callback=lambda error: pytest.fail(error), + roi_filenames=(path.name,), + require_source_metadata=True, + ) + ) diff --git a/tests/unit/test_spatial_graph_materialization.py b/tests/unit/test_spatial_graph_materialization.py index 6dc0145d26..0c0e70ef37 100644 --- a/tests/unit/test_spatial_graph_materialization.py +++ b/tests/unit/test_spatial_graph_materialization.py @@ -17,6 +17,7 @@ ObjectLabelsArtifactType, SpatialGraphArtifactType, ) +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.runtime_spatial_graph import ( @@ -340,7 +341,7 @@ def test_graph_roi_projection_preserves_paths_and_graph_features() -> None: assert roi_output.path == "/tmp/A01_neurite_graph_step3.graph.roi.zip" assert len(roi_output.content) == 3 - first_roi = roi_output.content[0] + first_roi = ROIArchiveSourceMetadata.geometry(roi_output.content)[0] assert isinstance(first_roi.shapes[0], PolylineShape) np.testing.assert_array_equal( first_roi.shapes[0].coordinates, From d092a03af34c3a4193084579d5e1a0f648c8102b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 03:38:30 -0400 Subject: [PATCH 049/125] fix(runtime): preserve whole-image domains across main-flow handoff Derive persisted image domains from original nominal producer members before physical leaf projection. Keep opaque images intact through copied cache reuse, reload, and artifact-only passthrough while preserving declared runtime/source-binding axes and legacy raw-array unstacking. Reject mixed joint domains and reuse existing bundle composition grammar. Resolves the false leading axis that sent physical 3D images into rank-four scientific execution (#419, #433). --- openhcs/core/aligned_image_payload.py | 38 ++ .../core/steps/function_output_manifest.py | 47 +- openhcs/core/steps/function_runtime.py | 204 ++++++--- tests/unit/test_function_artifact_outputs.py | 8 +- .../test_opaque_main_flow_image_domain.py | 420 ++++++++++++++++++ 5 files changed, 655 insertions(+), 62 deletions(-) create mode 100644 tests/unit/test_opaque_main_flow_image_domain.py diff --git a/openhcs/core/aligned_image_payload.py b/openhcs/core/aligned_image_payload.py index 115ece9341..6c0f82bfd2 100644 --- a/openhcs/core/aligned_image_payload.py +++ b/openhcs/core/aligned_image_payload.py @@ -1140,6 +1140,17 @@ def composition_payloads(self) -> tuple[Any, ...]: def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: return ImagePayloadMetadataCompositionMode.STACK + @property + def projected_output_composition_mode(self) -> ImagePayloadMetadataCompositionMode | None: + """Declare the outer runtime axis retained by projected output members.""" + return self.composition_metadata_mode + + def plane_axis_for_output_context( + self, context: AlignedImageSliceContext, + ) -> RuntimePlaneAxis | None: + """An explicitly aligned stack declares its outer runtime-slice domain.""" + return RuntimePlaneAxis.RUNTIME_SLICE + def composition_payload_metadata( self, metadata: ImagePayloadMetadata ) -> ImagePayloadMetadata: @@ -1286,6 +1297,33 @@ def composition_payloads(self) -> tuple[Any, ...]: def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: return ImagePayloadMetadataCompositionMode.BUNDLE + @property + def projected_output_composition_mode(self) -> ImagePayloadMetadataCompositionMode | None: + """Flatten declared inner runtime planes, retaining other named image domains.""" + if any( + image_payload_metadata(payload).plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + for payload in self.slices + ): + return ImagePayloadMetadataCompositionMode.STACK + if len(self.slices) == 1: + return None + return self.composition_metadata_mode + + def plane_axis_for_output_context( + self, context: AlignedImageSliceContext, + ) -> RuntimePlaneAxis | None: + """Resolve a named output's original inner domain before leaf projection.""" + payloads = tuple( + payload for payload, declared_context in zip(self.slices, self.slice_contexts, strict=True) + if declared_context == context + ) + if len(payloads) != 1: + raise ValueError( + "Named image output context requires exactly one original payload: " + f"{context!r}; found {len(payloads)}." + ) + return image_payload_metadata(payloads[0]).plane_axis + def composition_payload_metadata( self, metadata: ImagePayloadMetadata ) -> ImagePayloadMetadata: diff --git a/openhcs/core/steps/function_output_manifest.py b/openhcs/core/steps/function_output_manifest.py index ea93bbff1e..24ed7998dd 100644 --- a/openhcs/core/steps/function_output_manifest.py +++ b/openhcs/core/steps/function_output_manifest.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import ClassVar, Iterator, Sequence from weakref import WeakKeyDictionary @@ -23,6 +23,7 @@ ImagePayloadMetadata, image_payload_metadata, ) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.step_dependencies import StepInputDependencyKind from openhcs.core.steps.function_output_identity import ( @@ -113,6 +114,19 @@ class ProducedOutputSemantics(FunctionOutputIdentity): output_path: str relative_output_path: str image_metadata: ImagePayloadMetadata | None = None + main_flow_plane_axis: RuntimePlaneAxis | None = RuntimePlaneAxis.RUNTIME_SLICE + + def passed_through(self, plan: CompiledStepPlan) -> "ProducedOutputSemantics": + """Retain the image domain and physical identity under the next producer.""" + return replace( + self, + producer_identity=FunctionStepOutputProducerIdentityAuthority.build( + FunctionStepOutputProducerIdentityRequest.from_main_flow( + plan, self.output_context, + ) + ), + relative_output_path=source_path_identity(self.output_path).name, + ) def contextualize_image_payload( self, @@ -175,6 +189,7 @@ def from_output( output_identity: FunctionOutputIdentity, output_context: AlignedImageSliceContext | None = None, image_metadata: ImagePayloadMetadata | None = None, + main_flow_plane_axis: RuntimePlaneAxis | None = RuntimePlaneAxis.RUNTIME_SLICE, ) -> "ProducedOutputSemantics": output_path_text = str(output_path) if output_context is None: @@ -197,6 +212,7 @@ def from_output( Path(plan.output_dir), ), image_metadata=image_metadata, + main_flow_plane_axis=main_flow_plane_axis, ) @classmethod @@ -407,14 +423,27 @@ def producer_output_contexts_for_paths( ) -> tuple[AlignedImageSliceContext, ...]: """Return producer output contexts aligned to concrete input paths.""" - producer_records = self._selected_unique_producer_records_for(plan) - if producer_records is None: + records = self.producer_output_records_for_paths(plan, paths, parser) + if records is None: return tuple( AlignedImageSliceContext.anonymous_main_flow() for _path in paths ) + return tuple(record.output_context for record in records) + + def producer_output_records_for_paths( + self, + plan: CompiledStepPlan, + paths: Sequence[str], + parser: FilenameParser, + ) -> tuple[ProducedOutputSemantics, ...] | None: + """Resolve exact producer declarations in the requested physical path order.""" + + producer_records = self._selected_unique_producer_records_for(plan) + if producer_records is None: + return None path_index = ProducedPathRecordIndex.from_records(producer_records, parser) - contexts: list[AlignedImageSliceContext] = [] + records: list[ProducedOutputSemantics] = [] for path in paths: matching_records = path_index.matching_records(path) if len(matching_records) != 1: @@ -422,8 +451,8 @@ def producer_output_contexts_for_paths( "Expected one producer output context for input path " f"{path!r}, found {len(matching_records)}." ) - contexts.append(matching_records[0].output_context) - return tuple(contexts) + records.append(matching_records[0]) + return tuple(records) def filter_to_producer_paths( self, @@ -501,6 +530,12 @@ def _unique_output_path_records( ) -> tuple[ProducedOutputSemantics, ...]: records_by_path: dict[str, ProducedOutputSemantics] = {} for record in records: + existing = records_by_path.get(record.output_path) + if existing is not None and existing.main_flow_plane_axis is not record.main_flow_plane_axis: + raise ValueError( + "One produced output path cannot declare conflicting main-flow image axes: " + f"{record.output_path!r}." + ) records_by_path.setdefault(record.output_path, record) return tuple(records_by_path.values()) diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index 21f0ea6762..25825890b0 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -2421,6 +2421,7 @@ class PatternGroupOutputData: slices: list[RuntimeArrayData] slice_contexts: tuple[AlignedImageSliceContext, ...] = () stack_payload: RuntimeArrayData | None = None + main_flow_source: RuntimeArrayData | None = None def __post_init__(self) -> None: if not self.slice_contexts: @@ -2433,6 +2434,77 @@ def __post_init__(self) -> None: f"got {len(self.slice_contexts)} context(s) for {len(self.slices)} slice(s)." ) + @classmethod + def from_single_image( + cls, + value: RuntimeArrayData, + *, + slice_contexts: tuple[AlignedImageSliceContext, ...] = (), + memory_type: MemoryType, + device_id: int | None, + ) -> "PatternGroupOutputData": + """Copy one whole image for main-flow reuse without declaring a new axis.""" + copied_data = stack_runtime_slices( + (image_payload_data(value),), memory_type, device_id, + )[0] + mask = image_payload_mask(value) + copied_mask = ( + None if mask is None + else stack_runtime_slices((mask,), memory_type, device_id)[0] + ) + return cls( + slices=(value,), + slice_contexts=slice_contexts, + stack_payload=image_payload_metadata(value).replace_fields().payload_with( + copied_data, copied_mask, + ), + main_flow_source=value, + ) + + def plane_axis_for_output_context( + self, context: AlignedImageSliceContext, + ) -> RuntimePlaneAxis | None: + """Derive each persisted member's domain from its original nominal producer.""" + if isinstance(self.main_flow_source, AlignedImageStack): + return self.main_flow_source.plane_axis_for_output_context(context) + if isinstance(self.main_flow_source, ImagePayloadMetadataCarrier): + return image_payload_metadata(self.main_flow_source).plane_axis + return RuntimePlaneAxis.RUNTIME_SLICE + + def cache_payload_for_outputs( + self, + payloads: Sequence[RuntimeArrayData], + metadata: Sequence[ImagePayloadMetadata], + ) -> RuntimeArrayData | None: + """Keep the original whole-image domain while applying saved output context.""" + if self.stack_payload is None: + return None + data = image_payload_data(self.stack_payload) + if ( + len(payloads) == 1 + and self.plane_axis_for_output_context(self.slice_contexts[0]) is metadata[0].plane_axis + and np.shape(data) == np.shape(image_payload_data(payloads[0])) + ): + return metadata[0].replace_fields().payload_with( + data, image_payload_mask(self.stack_payload), + ) + if np.shape(data)[:1] != (len(payloads),): + raise ValueError( + "PatternGroupOutputData.stack_payload must match its declared " + f"output slice count: stack shape {np.shape(data)!r}, " + f"slice count {len(payloads)}." + ) + return stack_image_payload_context_from_metadata( + payloads, data, metadata, + metadata_mode=( + ImagePayloadMetadataCompositionMode.for_plane_axis( + image_payload_metadata(self.stack_payload).plane_axis, + ) + if isinstance(self.stack_payload, ImagePayloadMetadataCarrier) + else ImagePayloadMetadataCompositionMode.STACK + ), + ) + @classmethod def from_aligned_stack( cls, @@ -2444,15 +2516,32 @@ def from_aligned_stack( """Materialize correlated output slices through the alignment owner.""" projected_outputs = tuple(value.projected_output_slices()) payloads = [payload for payload, _context in projected_outputs] + metadata_mode = value.projected_output_composition_mode + if metadata_mode is None: + result = cls.from_single_image( + payloads[0], + slice_contexts=tuple(context for _payload, context in projected_outputs), + memory_type=memory_type, + device_id=device_id, + ) + result.main_flow_source = value + return result data = tuple(image_payload_data(payload) for payload in payloads) stack_payload = None - if len({tuple(np.shape(item)) for item in data}) == 1: - stacked = stack_runtime_slices(data, memory_type, device_id) - stack_payload = stack_image_payload_context( - payloads, - stacked, - metadata_mode=ImagePayloadMetadataCompositionMode.STACK, - ) + declared_axes = { + value.plane_axis_for_output_context(context) + for _payload, context in projected_outputs + } + if len(declared_axes) == 1 and len({tuple(np.shape(item)) for item in data}) == 1: + if metadata_mode is ImagePayloadMetadataCompositionMode.BUNDLE: + stack_payload = ImagePayloadBundleContext.from_payloads( + tuple(payloads), metadata_mode=metadata_mode, + ).compose() + else: + stacked = stack_runtime_slices(data, memory_type, device_id) + stack_payload = stack_image_payload_context( + payloads, stacked, metadata_mode=metadata_mode, + ) return cls( slices=payloads, slice_contexts=( @@ -2461,6 +2550,7 @@ def from_aligned_stack( else () ), stack_payload=stack_payload, + main_flow_source=value, ) def __iter__(self) -> Iterator[RuntimeArrayData]: @@ -2643,21 +2733,20 @@ def _record_main_flow_passthrough(self, matching_files: Sequence[str]) -> None: return plan = self.request.execution_plan parser = self.request.context.microscope_handler.parser - output_contexts = self._producer_output_contexts(matching_files) - records = tuple( - ProducedOutputSemantics.from_existing_main_flow_path( - plan, - self._input_memory_path(plan.input_dir, matching_file), - parser, - output_context=output_context, - ) - for matching_file, output_context in zip( - matching_files, - output_contexts, - strict=True, + manifest = step_output_manifest(self.request.context) + producer_records = manifest.producer_output_records_for_paths(plan, matching_files, parser) + if producer_records is None: + records = tuple( + ProducedOutputSemantics.from_existing_main_flow_path( + plan, + self._input_memory_path(plan.input_dir, matching_file), + parser, + ) + for matching_file in matching_files ) - ) - step_output_manifest(self.request.context).record_outputs(plan, records) + else: + records = tuple(record.passed_through(plan) for record in producer_records) + manifest.record_outputs(plan, records) def _producer_output_contexts( self, @@ -2757,6 +2846,15 @@ def _load_input_stack(self) -> PatternGroupData: matching_files=matching_files, source_projection=source_projection, ).runtime_context() + producer_records = ( + output_manifest.producer_output_records_for_paths( + plan, matching_files, context.microscope_handler.parser, + ) + if plan.main_input_dependency.kind is StepInputDependencyKind.STEP_OUTPUT + else None + ) + if producer_records and len({record.main_flow_plane_axis for record in producer_records}) != 1: + raise ValueError("One main-flow cohort cannot combine different declared image axes.") cached_stack = context.runtime_image_stack_cache.get( tuple(full_file_paths), memory_type=plan.input_memory_type, @@ -2793,7 +2891,30 @@ def _load_input_stack(self) -> PatternGroupData: f"Check file integrity and format compatibility." ) + if ( + producer_records + and len(producer_records) == 1 + and producer_records[0].main_flow_plane_axis is image_payload_metadata(raw_slices[0]).plane_axis + ): + main_data_stack = PatternGroupOutputData.from_single_image( + raw_slices[0], + memory_type=plan.input_memory_type, + device_id=plan.device_id_for(plan.input_memory_type), + ).stack_payload + return PatternGroupData( + matching_files=matching_files, + main_data_stack=main_data_stack, + source_binding_context=source_binding_context, + ) + metadata_mode = ImagePayloadMetadataCompositionMode.STACK + if producer_records: + declared_axis = producer_records[0].main_flow_plane_axis + metadata_mode = ( + ImagePayloadMetadataCompositionMode.BUNDLE + if declared_axis is None + else ImagePayloadMetadataCompositionMode.for_plane_axis(declared_axis) + ) if source_projection is not None and workspace_source_lookups: metadata_mode = source_projection.payload_composition_mode( workspace_source_lookups @@ -3117,16 +3238,8 @@ def _validate_and_unstack( and processed_stack.metadata.plane_axis is None ): output_context = self._unwrapped_main_flow_output_context() - scalar_data = image_payload_data(processed_stack) - stacked_data = stack_runtime_slices( - (scalar_data,), - self.request.execution_plan.output_memory_type, - self.request.execution_plan.device_id_for( - self.request.execution_plan.output_memory_type - ), - ) - return PatternGroupOutputData( - slices=(processed_stack,), + return PatternGroupOutputData.from_single_image( + processed_stack, slice_contexts=( (output_context,) if output_context is not None @@ -3136,10 +3249,9 @@ def _validate_and_unstack( else () ) ), - stack_payload=stack_image_payload_context( - (processed_stack,), - stacked_data, - metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + memory_type=self.request.execution_plan.output_memory_type, + device_id=self.request.execution_plan.device_id_for( + self.request.execution_plan.output_memory_type ), ) if isinstance(processed_stack, AlignedImageStack): @@ -3229,6 +3341,7 @@ def _validate_and_unstack( slices=output_payloads, slice_contexts=slice_contexts, stack_payload=processed_stack, + main_flow_source=processed_stack, ) def _unwrapped_main_flow_output_context( @@ -3336,6 +3449,7 @@ def _save_outputs( output_identity, output_context=output_context, image_metadata=output_metadata, + main_flow_plane_axis=output_data.plane_axis_for_output_context(output_context), ) if output_directory_exists and context.filemanager.exists( @@ -3373,24 +3487,10 @@ def _save_outputs( output_paths_batch, Backend.MEMORY.value, ) - stack_payload_data = ( - image_payload_data(output_data.stack_payload) - if output_data.stack_payload is not None - else None + stack_payload = output_data.cache_payload_for_outputs( + output_payloads, output_payload_metadata, ) - if output_data.stack_payload is not None: - if np.shape(stack_payload_data)[:1] != (len(output_payloads),): - raise ValueError( - "PatternGroupOutputData.stack_payload must match its declared " - f"output slice count: stack shape {np.shape(stack_payload_data)!r}, " - f"slice count {len(output_payloads)}." - ) - stack_payload = stack_image_payload_context_from_metadata( - output_payloads, - stack_payload_data, - output_payload_metadata, - metadata_mode=ImagePayloadMetadataCompositionMode.STACK, - ) + if stack_payload is not None: context.runtime_image_stack_cache.store( tuple(output_paths_batch), memory_type=self.request.execution_plan.output_memory_type, diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index 5cda2912d1..7bbeb49d34 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -1704,7 +1704,7 @@ def enhance(image, *, runtime): } -def test_pattern_group_runtime_stacks_nominal_scalar_rgb_output_as_one_slice(): +def test_pattern_group_runtime_retains_nominal_scalar_rgb_output_as_one_image(): scalar_output = ImagePayloadMetadata( source_path="/input/A01_s1_w3.tif", source_component_metadata={ @@ -1767,10 +1767,10 @@ def rgb_output(image): artifact_kind=ImageArtifactType.value, ), ) - assert image_payload_data(output.stack_payload).shape == (1, 2, 3, 3) + assert image_payload_data(output.stack_payload).shape == (2, 3, 3) stack_metadata = image_payload_metadata(output.stack_payload) - assert stack_metadata.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE - assert stack_metadata.source_channel_axis == 3 + assert stack_metadata.plane_axis is None + assert stack_metadata.source_channel_axis == -1 def test_pattern_group_runtime_uses_declared_output_slice_cardinality(): diff --git a/tests/unit/test_opaque_main_flow_image_domain.py b/tests/unit/test_opaque_main_flow_image_domain.py new file mode 100644 index 0000000000..619c93677d --- /dev/null +++ b/tests/unit/test_opaque_main_flow_image_domain.py @@ -0,0 +1,420 @@ +"""Whole image domains survive the producer/cache/next-load handoff.""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pytest + +from openhcs.constants.constants import MEMORY_TYPE_NUMPY +from openhcs.core.compiled_step_plan import CompiledStepPlan +from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.function_patterns import compile_function_pattern +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.runtime_slice_projection import RuntimeSliceProjection +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.core.steps.function_runtime import PatternGroupData, PatternGroupRuntime +from openhcs.core.aligned_image_payload import AlignedImageStack, ImageOutputBundle, AlignedImageSliceContext +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.steps.function_output_manifest import step_output_manifest, StepOutputManifestStore +from openhcs.core.source_bindings import CompiledSourceBindingPlan, SourceBindingRuntimeContext +from openhcs.core.steps.function_runtime import SourceBindingRuntimeContextRequest +from openhcs.core.step_dependencies import StepInputDependency +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes + + +def _identity(image): + return image + + +def _runtime(): + pattern = compile_function_pattern(_identity, {}, {}) + runtime = object.__new__(PatternGroupRuntime) + runtime.request = SimpleNamespace( + compiled_group=pattern.default_group, + execution_plan=CompiledStepPlan( + step_index=0, + step_name="WholeImage", + step_type="FunctionStep", + axis_id="A01", + output_memory_type=MEMORY_TYPE_NUMPY, + input_memory_type=MEMORY_TYPE_NUMPY, + variable_components=(), + artifact_inputs={}, + artifact_outputs={}, + execution_group_scope=ComponentGroupScope.ungrouped(), + ), + component_key=None, + ) + return runtime + + +@pytest.mark.parametrize( + ("shape", "channel_axis"), + [((4, 5), None), ((4, 5, 3), -1), ((3, 4, 5), None), ((1, 4, 5), None)], + ids=("plane", "rgb", "volume", "one-z-volume"), +) +def test_opaque_whole_image_is_copied_without_a_new_execution_axis(shape, channel_axis): + pixels = np.arange(np.prod(shape), dtype=np.float32).reshape(shape) + mask_shape = shape[:2] if channel_axis is not None else shape + mask = np.ones(mask_shape, dtype=bool) + metadata = ImagePayloadMetadata( + source_path="/input/A01_s001_w1_z001_t001.tif", + source_component_metadata={"well": "A01", "site": "1", "channel": "1"}, + source_image_names=("WholeImage",), + source_channel_axis=channel_axis, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(12, 13), + ), + ) + payload = metadata.payload_with(pixels, mask) + output = _runtime()._validate_and_unstack( + payload, + PatternGroupData( + matching_files=["/input/A01_s001_w1_z001_t001.tif", "/input/A01_s001_w1_z002_t001.tif"], + main_data_stack=np.zeros((2, 4, 5), dtype=np.float32), + ), + ) + assert len(output.slices) == 1 + assert output.slices[0] is payload + cached = output.stack_payload + assert image_payload_data(cached).shape == shape + assert image_payload_metadata(cached) == metadata + np.testing.assert_array_equal(image_payload_data(cached), pixels) + np.testing.assert_array_equal(image_payload_mask(cached), mask) + assert not np.shares_memory(image_payload_data(cached), pixels) + assert not np.shares_memory(image_payload_mask(cached), mask) + assert image_payload_metadata(cached) is not metadata + assert RuntimeSliceProjection.full_stack_value(cached) is cached + image_payload_data(cached).flat[0] = -100 + image_payload_mask(cached).flat[0] = False + assert pixels.flat[0] == 0 + assert mask.flat[0] + image_payload_metadata(cached).source_channel_axis = 0 + assert metadata.source_channel_axis == channel_axis + + +@pytest.mark.parametrize("count", [1, 3]) +def test_declared_runtime_axis_retains_its_exact_count(count): + pixels = np.arange(count * 4 * 5, dtype=np.float32).reshape(count, 4, 5) + payload = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE).payload_with(pixels) + output = _runtime()._validate_and_unstack( + payload, + PatternGroupData(matching_files=["source-1.tif", "source-2.tif"], main_data_stack=pixels), + ) + assert len(output.slices) == count + assert output.stack_payload is payload + assert image_payload_data(output.stack_payload).shape == (count, 4, 5) + assert image_payload_metadata(output.stack_payload).plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + + +@pytest.mark.parametrize("named", [False, True]) +@pytest.mark.parametrize("axis", [None, RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.SOURCE_BINDING]) +def test_nominal_alignment_owner_preserves_single_output_topology(named, axis): + shape = (3, 4, 5) if axis is None else (1, 4, 5) + payload = ImagePayloadMetadata(plane_axis=axis).payload_with(np.ones(shape, dtype=np.float32)) + context = AlignedImageSliceContext.main_flow("Image") + value = (ImageOutputBundle if named else AlignedImageStack)((payload,), (context,)) + output = _runtime()._validate_and_unstack( + value, PatternGroupData(matching_files=["source.tif"], main_data_stack=payload), + ) + if named and axis is not RuntimePlaneAxis.RUNTIME_SLICE: + expected_shape, expected_axis = shape, axis + else: + expected_shape = (1, 4, 5) if axis is RuntimePlaneAxis.RUNTIME_SLICE else (1, *shape) + expected_axis = RuntimePlaneAxis.RUNTIME_SLICE + assert image_payload_data(output.stack_payload).shape == expected_shape + assert image_payload_metadata(output.stack_payload).plane_axis is expected_axis + + +@pytest.mark.parametrize("count", [1, 3]) +def test_raw_array_fallback_keeps_explicit_runtime_unstack_contract(count): + runtime = _runtime() + runtime.request.context = ProcessingContext(axis_id="A01") + runtime.request.context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + pixels = np.arange(count * 4 * 5, dtype=np.float32).reshape(count, 4, 5) + output = runtime._validate_and_unstack( + pixels, PatternGroupData(matching_files=[f"source-{i}.tif" for i in range(count)], main_data_stack=pixels), + ) + cached = output.cache_payload_for_outputs(output.slices, [image_payload_metadata(p) for p in output.slices]) + assert image_payload_data(cached).shape == (count, 4, 5) + assert image_payload_metadata(cached).plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + + +class _MemoryFiles: + def __init__(self): + self.values = {} + + def exists(self, path, backend): + return path in self.values or any(value_path.startswith(path.rstrip("/") + "/") for value_path in self.values) + + def ensure_directory(self, path, backend): + pass + + def save_batch(self, payloads, paths, backend): + self.values.update(zip(paths, payloads, strict=True)) + + def load_batch(self, paths, backend): + return [self.values[path] for path in paths] + + def delete(self, path, backend): + del self.values[path] + + +@pytest.mark.parametrize("cache_hit", [False, True]) +@pytest.mark.parametrize("axis", [None, RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.SOURCE_BINDING]) +@pytest.mark.parametrize("shape", [(4, 5), (4, 5, 3), (3, 4, 5), (1, 4, 5)]) +def test_save_and_next_load_preserve_domain_and_independent_cache(tmp_path, monkeypatch, cache_hit, axis, shape, raw_named=False): + pixels = np.arange(np.prod(shape), dtype=np.float32).reshape(shape) + channel_axis = -1 if shape == (4, 5, 3) else None + if axis is not None: + pixels = pixels[None] + channel_axis = 3 if channel_axis is not None else None + mask = np.ones(pixels.shape[:-1] if channel_axis is not None else pixels.shape, dtype=bool) + path = "/input/A01_s001_w1_z001_t001.tif" + components = {"well": "A01", "site": "1", "channel": "1", "z_index": "1", "timepoint": "1"} + metadata = ImagePayloadMetadata( + source_path=path, source_component_metadata=components, source_image_names=("WholeImage",), + source_channel_axis=channel_axis, plane_axis=axis, + source_image_provenance_planes=( + SourceImageProvenancePlanes.from_components(paths=(path,), component_metadata=(components,)) + if axis is not None else SourceImageProvenancePlanes() + ), + ) + payload = pixels if raw_named else metadata.payload_with(pixels, mask) + if raw_named: + mask = None + producer = _runtime() + pattern = compile_function_pattern(_identity, {}, {}) + plan = replace(producer.request.execution_plan, output_dir=tmp_path, step_scope_id="whole-producer", compiled_function_pattern=pattern) + producer.request.execution_plan = plan + files = _MemoryFiles() + context = ProcessingContext(axis_id="A01", filemanager=files) + context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + producer.request.context = context + producer.pattern_repr = "whole" + output = producer._validate_and_unstack( + ImageOutputBundle((payload,), (AlignedImageSliceContext.main_flow("WholeImage"),)) if axis is not None or raw_named else payload, + PatternGroupData(matching_files=[path, "/input/A01_s001_w1_z002_t001.tif"], main_data_stack=pixels), + ) + records = producer._save_outputs(output, [path]) + manifest = step_output_manifest(context) + manifest.record_outputs(plan, records) + assert records[0].main_flow_plane_axis is axis + physical = files.values[records[0].output_path] + assert np.shares_memory(image_payload_data(physical), pixels) + if not cache_hit: + context.runtime_image_stack_cache.clear() + consumer_plan = replace( + plan, step_index=1, step_scope_id="whole-consumer", input_dir=tmp_path, + main_input_dependency=StepInputDependency.step_output(source_step_index=0, source_step_scope_id=plan.step_scope_id), + ) + consumer = PatternGroupRuntime(SimpleNamespace( + context=context, execution_plan=consumer_plan, compiled_group=pattern.default_group, + source_binding_plan=CompiledSourceBindingPlan.empty(), component_key=None, + pattern_group_info="A01_s001_w1_z{iii}_t001.tif", + )) + monkeypatch.setattr(consumer, "source_workspace_projection_authority", lambda: SimpleNamespace(projection_if_available=lambda: None)) + monkeypatch.setattr(SourceBindingRuntimeContextRequest, "from_context", classmethod(lambda cls, **kwargs: SimpleNamespace(runtime_context=SourceBindingRuntimeContext.empty))) + loaded = consumer._load_input_stack().main_data_stack + assert image_payload_data(loaded).shape == pixels.shape + assert image_payload_metadata(loaded).plane_axis is axis + np.testing.assert_array_equal(image_payload_data(loaded), pixels) + np.testing.assert_array_equal(image_payload_mask(loaded), mask) + assert not np.shares_memory(image_payload_data(loaded), pixels) + if mask is not None: + assert not np.shares_memory(image_payload_mask(loaded), mask) + assert image_payload_metadata(loaded) is not image_payload_metadata(physical) + image_payload_metadata(loaded).source_provenance = image_payload_metadata(loaded).source_provenance.with_source_image_names(("Changed",)) + assert image_payload_metadata(physical).source_image_names == ("WholeImage",) + image_payload_data(loaded).flat[0] = -100 + if mask is not None: + image_payload_mask(loaded).flat[0] = False + assert mask.flat[0] + assert pixels.flat[0] == 0 + consumer._record_main_flow_passthrough([records[0].relative_output_path]) + (passed,) = manifest.produced_records_for(consumer_plan) + assert passed.main_flow_plane_axis is axis + assert passed.image_metadata is records[0].image_metadata + + +@pytest.mark.parametrize("runtime_count", [1, 3]) +def test_mixed_named_outputs_retain_each_original_axis_before_projection(runtime_count): + contexts = (AlignedImageSliceContext.main_flow("Planes"), AlignedImageSliceContext.main_flow("Volume")) + planes = ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.RUNTIME_SLICE).payload_with(np.ones((runtime_count, 4, 5), dtype=np.float32)) + volume = ImagePayloadMetadata().payload_with(np.ones((4, 5) if runtime_count == 1 else (2, 4, 5), dtype=np.float32)) + bundle = ImageOutputBundle((planes, volume), contexts) + output = _runtime()._validate_and_unstack(bundle, PatternGroupData(matching_files=[], main_data_stack=planes)) + assert output.stack_payload is None + assert [output.plane_axis_for_output_context(c) for c in output.slice_contexts] == [RuntimePlaneAxis.RUNTIME_SLICE] * runtime_count + [None] + assert output.main_flow_source is bundle + + +def test_passed_through_record_preserves_collapsed_and_filename_coordinates(tmp_path): + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + record = ProducedOutputSemantics.from_output( + plan, tmp_path / "A01_s001_w1_z001_t001.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, filename_component_values={"well": "A01", "z_index": 1}, extension=".tif", source="test"), + main_flow_plane_axis=None, + ) + passed = record.passed_through(replace(plan, step_index=1, step_scope_id="next")) + assert passed.component_values == record.component_values + assert passed.filename_component_values == record.filename_component_values + assert passed.main_flow_plane_axis is None + + +def test_duplicate_physical_path_rejects_contradictory_whole_axis(tmp_path): + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + opaque = ProducedOutputSemantics.from_output( + plan, tmp_path / "A01_s001_w1_z001_t001.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, extension=".tif", source="test"), + main_flow_plane_axis=None, + ) + with pytest.raises(ValueError, match="conflicting main-flow image axes"): + StepOutputManifestStore._unique_output_path_records((opaque, replace(opaque, main_flow_plane_axis=RuntimePlaneAxis.RUNTIME_SLICE))) + + +@pytest.mark.parametrize("cache_hit", [False, True]) +@pytest.mark.parametrize("shape", [(4, 5), (4, 5, 3), (3, 4, 5), (1, 4, 5)]) +def test_named_raw_image_actual_save_and_reload_retains_opaque_domain(tmp_path, monkeypatch, cache_hit, shape): + test_save_and_next_load_preserve_domain_and_independent_cache( + tmp_path, monkeypatch, cache_hit, None, shape, raw_named=True, + ) + + +def test_record_axis_survives_replace_and_pickle(tmp_path): + import pickle + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + for axis in (None, RuntimePlaneAxis.RUNTIME_SLICE, RuntimePlaneAxis.SOURCE_BINDING): + record = ProducedOutputSemantics.from_output( + plan, tmp_path / "A01_s001_w1_z001_t001.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, extension=".tif", source="test"), + main_flow_plane_axis=axis, + ) + assert replace(record).main_flow_plane_axis is axis + restored = pickle.loads(pickle.dumps(record)) + assert restored == record + assert restored.main_flow_plane_axis is axis + + +def _selected_pattern(name): + from openhcs.core.artifacts import ArtifactSpec, ImageArtifactType + from openhcs.core.pipeline.function_contracts import artifact_inputs + from openhcs.core.function_patterns import InvocationArtifactInputEdgePlan, InvocationArtifactInputProjectionKey + spec = ArtifactSpec.input(name, ImageArtifactType) + + @artifact_inputs(spec) + def consume(image): + return image + + pattern = compile_function_pattern(consume, {}, {}) + invocation = pattern.default_group.invocations[0] + invocation = invocation.with_artifact_input_edges((InvocationArtifactInputEdgePlan( + key=InvocationArtifactInputProjectionKey(invocation_key=invocation.key, input_index=0), + spec=spec, storage_plan=None, projection=None, consumes_main_flow=True, + ),)) + return replace(pattern, groups=(replace(pattern.default_group, invocations=(invocation,)),)) + + +@pytest.mark.parametrize("cache_hit", [False, True]) +def test_saved_mixed_named_cohort_rejects_joint_load_and_preserves_selected_domains(tmp_path, monkeypatch, cache_hit): + from openhcs.core.artifacts import ImageArtifactType + source = "/input/A01_s001_w1_z001_t001.tif" + components = {"well": "A01", "site": "1", "channel": "1", "z_index": "1", "timepoint": "1"} + contexts = tuple(AlignedImageSliceContext.main_flow(name, artifact_kind=ImageArtifactType.value) for name in ("Planes", "Volume")) + scalar_metadata = ImagePayloadMetadata(source_path=source, source_component_metadata=components) + declared_metadata = scalar_metadata.replace_fields( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components(paths=(source,), component_metadata=(components,)), + ) + planes = declared_metadata.payload_with(np.arange(20, dtype=np.float32).reshape(1, 4, 5)) + opaque_pixels = np.arange(20, dtype=np.float32).reshape(4, 5) + 100 + opaque = scalar_metadata.payload_with(opaque_pixels) + producer = _runtime() + pattern = compile_function_pattern(_identity, {}, {}) + plan = replace(producer.request.execution_plan, output_dir=tmp_path, step_scope_id="mixed-producer", compiled_function_pattern=pattern) + producer.request.execution_plan = plan + files = _MemoryFiles() + context = ProcessingContext(axis_id="A01", filemanager=files) + context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + producer.request.context = context + producer.pattern_repr = "mixed" + loaded = PatternGroupData(matching_files=[source, source], main_data_stack=planes) + + # A prior homogeneous producer caches the same physical paths. Mixed replacement + # must invalidate that cache rather than leave a false joint runtime domain. + old_bundle = ImageOutputBundle((planes, declared_metadata.payload_with(opaque_pixels[None])), contexts) + old_records = producer._save_outputs(producer._validate_and_unstack(old_bundle, loaded), [source, source]) + assert context.runtime_image_stack_cache.get(tuple(r.output_path for r in old_records), memory_type=MEMORY_TYPE_NUMPY) is not None + bundle = ImageOutputBundle((planes, opaque), contexts) + output = producer._validate_and_unstack(bundle, loaded) + records = producer._save_outputs(output, [source, source]) + manifest = step_output_manifest(context) + manifest.record_outputs(plan, records) + assert tuple(r.output_path for r in records) == tuple(r.output_path for r in old_records) + assert [r.main_flow_plane_axis for r in records] == [RuntimePlaneAxis.RUNTIME_SLICE, None] + assert context.runtime_image_stack_cache.get(tuple(r.output_path for r in records), memory_type=MEMORY_TYPE_NUMPY) is None + monkeypatch.setattr(SourceBindingRuntimeContextRequest, "from_context", classmethod(lambda cls, **kwargs: SimpleNamespace(runtime_context=SourceBindingRuntimeContext.empty))) + + def consumer_for(compiled): + consumer_plan = replace(plan, step_index=1, step_scope_id="mixed-consumer", input_dir=tmp_path, + compiled_function_pattern=compiled, + main_input_dependency=StepInputDependency.step_output(source_step_index=0, source_step_scope_id=plan.step_scope_id)) + consumer = PatternGroupRuntime(SimpleNamespace(context=context, execution_plan=consumer_plan, + compiled_group=compiled.default_group, source_binding_plan=CompiledSourceBindingPlan.empty(), + component_key=None, pattern_group_info="A01_s001_w1_z{iii}_t001.tif")) + monkeypatch.setattr(consumer, "source_workspace_projection_authority", lambda: SimpleNamespace(projection_if_available=lambda: None)) + return consumer + + with pytest.raises(ValueError, match="different declared image axes"): + consumer_for(pattern)._load_input_stack() + for name, expected, axis in (("Planes", image_payload_data(planes), RuntimePlaneAxis.RUNTIME_SLICE), ("Volume", opaque_pixels, None)): + selected = consumer_for(_selected_pattern(name)) + actual = selected._load_input_stack().main_data_stack + np.testing.assert_array_equal(image_payload_data(actual), expected) + assert image_payload_metadata(actual).plane_axis is axis + if not cache_hit: + context.runtime_image_stack_cache.clear() + else: + selected_path = next(r.output_path for r in records if r.producer_identity.output_key == name) + context.runtime_image_stack_cache.store((selected_path,), memory_type=MEMORY_TYPE_NUMPY, stack=actual) + repeated = selected._load_input_stack().main_data_stack + np.testing.assert_array_equal(image_payload_data(repeated), expected) + assert image_payload_metadata(repeated).plane_axis is axis + + +def test_multiple_unprojected_named_binding_domains_fail_existing_bundle_grammar(): + contexts = tuple(AlignedImageSliceContext.main_flow(name) for name in ("First", "Second")) + payloads = tuple(ImagePayloadMetadata(plane_axis=RuntimePlaneAxis.SOURCE_BINDING).payload_with( + np.ones((1, 4, 5), dtype=np.float32)) for _ in contexts) + with pytest.raises(ValueError, match="plane axis to be projected"): + _runtime()._validate_and_unstack(ImageOutputBundle(payloads, contexts), PatternGroupData(matching_files=[], main_data_stack=payloads[0])) + + +def test_opaque_named_bundle_uses_existing_same_slice_mask_composition(): + from openhcs.core.aligned_image_payload import ImagePayloadBundleContext + contexts = tuple(AlignedImageSliceContext.main_flow(name) for name in ("First", "Second")) + first_mask = np.ones((4, 5), dtype=bool) + second_mask = first_mask.copy() + second_mask[0, 0] = False + payloads = tuple(ImagePayloadMetadata(source_image_names=(name,)).payload_with( + np.ones((4, 5), dtype=np.float32) * i, mask) + for i, (name, mask) in enumerate(zip(("First", "Second"), (first_mask, second_mask), strict=True))) + output = _runtime()._validate_and_unstack(ImageOutputBundle(payloads, contexts), PatternGroupData(matching_files=[], main_data_stack=payloads[0])) + expected = ImagePayloadBundleContext.from_payloads(payloads).compose() + assert image_payload_metadata(output.stack_payload).plane_axis is RuntimePlaneAxis.SOURCE_BINDING + np.testing.assert_array_equal(image_payload_data(output.stack_payload), image_payload_data(expected)) + np.testing.assert_array_equal(image_payload_mask(output.stack_payload), image_payload_mask(expected)) From 6dd47dd7e97784dc491b1fc47cfbee3e27cddb38 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 03:51:02 -0400 Subject: [PATCH 050/125] refactor(runtime): let input data own main-flow image composition Move declared cohort admissibility and loaded-image composition from the oversized runtime loader to PatternGroupData. Reuse ImagePayloadStackComposition for independently copied whole images across input and output owners, retaining plan device resolution order. Keep exact producer-domain records as the authority and remove the input-to-output factory dependency. Related to #419 and #433. --- openhcs/core/aligned_image_payload.py | 20 +++ openhcs/core/steps/function_runtime.py | 135 ++++++++++-------- .../test_opaque_main_flow_image_domain.py | 25 ++++ 3 files changed, 121 insertions(+), 59 deletions(-) diff --git a/openhcs/core/aligned_image_payload.py b/openhcs/core/aligned_image_payload.py index 6c0f82bfd2..98451b0e19 100644 --- a/openhcs/core/aligned_image_payload.py +++ b/openhcs/core/aligned_image_payload.py @@ -423,6 +423,26 @@ def composition_payloads(self) -> tuple[Any, ...]: ... @abstractmethod def composition_metadata_mode(self) -> ImagePayloadMetadataCompositionMode: ... + @staticmethod + def copy_whole_image( + value: RuntimeArrayData, + *, + memory_type: str, + device_id: int | None, + ) -> RuntimeArrayData: + """Copy pixels, mask and metadata without adding a composition axis.""" + copied_data = stack_runtime_slices( + (image_payload_data(value),), memory_type, device_id, + )[0] + mask = image_payload_mask(value) + copied_mask = ( + None if mask is None + else stack_runtime_slices((mask,), memory_type, device_id)[0] + ) + return image_payload_metadata(value).replace_fields().payload_with( + copied_data, copied_mask, + ) + def composition_source_metadata(self) -> tuple[ImagePayloadMetadata, ...]: return tuple( self.composition_payload_metadata(image_payload_metadata(payload)) diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index 25825890b0..e0fbc70fea 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -74,6 +74,7 @@ AlignedImageStack, AlignedImageSliceContext, ImagePayloadBundleContext, + ImagePayloadStackComposition, ImageOutputBundle, flatten_aligned_image_payload_slices, stack_image_payload_context, @@ -1576,6 +1577,69 @@ class PatternGroupData: ) + @staticmethod + def validate_main_flow_cohort( + producer_records: Sequence[ProducedOutputSemantics] | None, + ) -> None: + """One input cohort has one declared whole-image composition domain.""" + if producer_records and len({record.main_flow_plane_axis for record in producer_records}) != 1: + raise ValueError("One main-flow cohort cannot combine different declared image axes.") + + @classmethod + def from_loaded_images( + cls, + matching_files: list[str], + payloads: Sequence[RuntimeArrayData], + *, + producer_records: Sequence[ProducedOutputSemantics] | None, + source_binding_context: SourceBindingRuntimeContext, + execution_plan: CompiledStepPlan, + source_projection: VirtualWorkspaceSourceProjection | None, + workspace_source_lookups: Sequence[VirtualWorkspacePathLookup], + ) -> "PatternGroupData": + """Compose a selected admissible input cohort in its declared image domain.""" + if ( + producer_records + and len(producer_records) == 1 + and producer_records[0].main_flow_plane_axis is image_payload_metadata(payloads[0]).plane_axis + ): + main_data_stack = ImagePayloadStackComposition.copy_whole_image( + payloads[0], memory_type=execution_plan.input_memory_type, + device_id=execution_plan.device_id_for(execution_plan.input_memory_type), + ) + else: + metadata_mode = ImagePayloadMetadataCompositionMode.STACK + if producer_records: + declared_axis = producer_records[0].main_flow_plane_axis + metadata_mode = ( + ImagePayloadMetadataCompositionMode.BUNDLE + if declared_axis is None + else ImagePayloadMetadataCompositionMode.for_plane_axis(declared_axis) + ) + if source_projection is not None and workspace_source_lookups: + metadata_mode = source_projection.payload_composition_mode( + workspace_source_lookups + ) + if metadata_mode is ImagePayloadMetadataCompositionMode.STACK: + main_data_stack = stack_runtime_slices( + tuple(image_payload_data(payload) for payload in payloads), + execution_plan.input_memory_type, + execution_plan.device_id_for(execution_plan.input_memory_type), + ) + main_data_stack = stack_image_payload_context( + payloads, main_data_stack, metadata_mode=metadata_mode, + ) + elif metadata_mode is ImagePayloadMetadataCompositionMode.BUNDLE: + main_data_stack = ImagePayloadBundleContext.from_payloads( + tuple(payloads), metadata_mode=metadata_mode, + ).compose() + return cls( + matching_files=matching_files, + main_data_stack=main_data_stack, + source_binding_context=source_binding_context, + ) + + @dataclass(frozen=True, slots=True) class OutputPathBatchEntry: """Resolved identity for one output path in a runtime save batch.""" @@ -2444,19 +2508,11 @@ def from_single_image( device_id: int | None, ) -> "PatternGroupOutputData": """Copy one whole image for main-flow reuse without declaring a new axis.""" - copied_data = stack_runtime_slices( - (image_payload_data(value),), memory_type, device_id, - )[0] - mask = image_payload_mask(value) - copied_mask = ( - None if mask is None - else stack_runtime_slices((mask,), memory_type, device_id)[0] - ) return cls( slices=(value,), slice_contexts=slice_contexts, - stack_payload=image_payload_metadata(value).replace_fields().payload_with( - copied_data, copied_mask, + stack_payload=ImagePayloadStackComposition.copy_whole_image( + value, memory_type=memory_type, device_id=device_id, ), main_flow_source=value, ) @@ -2853,8 +2909,7 @@ def _load_input_stack(self) -> PatternGroupData: if plan.main_input_dependency.kind is StepInputDependencyKind.STEP_OUTPUT else None ) - if producer_records and len({record.main_flow_plane_axis for record in producer_records}) != 1: - raise ValueError("One main-flow cohort cannot combine different declared image axes.") + PatternGroupData.validate_main_flow_cohort(producer_records) cached_stack = context.runtime_image_stack_cache.get( tuple(full_file_paths), memory_type=plan.input_memory_type, @@ -2891,53 +2946,15 @@ def _load_input_stack(self) -> PatternGroupData: f"Check file integrity and format compatibility." ) - if ( - producer_records - and len(producer_records) == 1 - and producer_records[0].main_flow_plane_axis is image_payload_metadata(raw_slices[0]).plane_axis - ): - main_data_stack = PatternGroupOutputData.from_single_image( - raw_slices[0], - memory_type=plan.input_memory_type, - device_id=plan.device_id_for(plan.input_memory_type), - ).stack_payload - return PatternGroupData( - matching_files=matching_files, - main_data_stack=main_data_stack, - source_binding_context=source_binding_context, - ) - - metadata_mode = ImagePayloadMetadataCompositionMode.STACK - if producer_records: - declared_axis = producer_records[0].main_flow_plane_axis - metadata_mode = ( - ImagePayloadMetadataCompositionMode.BUNDLE - if declared_axis is None - else ImagePayloadMetadataCompositionMode.for_plane_axis(declared_axis) - ) - if source_projection is not None and workspace_source_lookups: - metadata_mode = source_projection.payload_composition_mode( - workspace_source_lookups - ) - if metadata_mode is ImagePayloadMetadataCompositionMode.STACK: - raw_slice_data = tuple( - image_payload_data(slice_data) for slice_data in raw_slices - ) - main_data_stack = stack_runtime_slices( - raw_slice_data, - plan.input_memory_type, - plan.device_id_for(plan.input_memory_type), - ) - main_data_stack = stack_image_payload_context( - raw_slices, - main_data_stack, - metadata_mode=metadata_mode, - ) - elif metadata_mode is ImagePayloadMetadataCompositionMode.BUNDLE: - main_data_stack = ImagePayloadBundleContext.from_payloads( - tuple(raw_slices), - metadata_mode=metadata_mode, - ).compose() + return PatternGroupData.from_loaded_images( + matching_files, + raw_slices, + producer_records=producer_records, + source_binding_context=source_binding_context, + execution_plan=plan, + source_projection=source_projection, + workspace_source_lookups=workspace_source_lookups, + ) else: main_data_stack = cached_stack diff --git a/tests/unit/test_opaque_main_flow_image_domain.py b/tests/unit/test_opaque_main_flow_image_domain.py index 619c93677d..6553bf96ad 100644 --- a/tests/unit/test_opaque_main_flow_image_domain.py +++ b/tests/unit/test_opaque_main_flow_image_domain.py @@ -418,3 +418,28 @@ def test_opaque_named_bundle_uses_existing_same_slice_mask_composition(): assert image_payload_metadata(output.stack_payload).plane_axis is RuntimePlaneAxis.SOURCE_BINDING np.testing.assert_array_equal(image_payload_data(output.stack_payload), image_payload_data(expected)) np.testing.assert_array_equal(image_payload_mask(output.stack_payload), image_payload_mask(expected)) + + +def test_input_bundle_composition_retains_lazy_plan_device_resolution(tmp_path, monkeypatch): + from openhcs.core.steps.function_output_manifest import ProducedOutputSemantics + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity + plan = replace(_runtime().request.execution_plan, output_dir=tmp_path) + records = tuple(ProducedOutputSemantics.from_output( + plan, tmp_path / f"A01_s001_w1_z001_t001_{name}.tif", + FunctionOutputIdentity(component_values={"well": "A01"}, extension=".tif", source="test"), + main_flow_plane_axis=None, + ) for name in ("First", "Second")) + payloads = tuple(ImagePayloadMetadata(source_image_names=(name,)).payload_with( + np.ones((4, 5), dtype=np.float32) * i) for i, name in enumerate(("First", "Second"))) + + def unexpected_device_resolution(self, memory_type): + raise AssertionError("BUNDLE composition must retain its own memory allocation authority") + + monkeypatch.setattr(CompiledStepPlan, "device_id_for", unexpected_device_resolution) + output = PatternGroupData.from_loaded_images( + [r.output_path for r in records], payloads, + producer_records=records, source_binding_context=SourceBindingRuntimeContext.empty(), + execution_plan=plan, source_projection=None, workspace_source_lookups=(), + ) + assert image_payload_metadata(output.main_data_stack).plane_axis is RuntimePlaneAxis.SOURCE_BINDING + np.testing.assert_array_equal(image_payload_data(output.main_data_stack), np.stack([image_payload_data(p) for p in payloads])) From 130c03a26684a2f3b6eb262e3ede7b2ce31bed64 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 04:13:34 -0400 Subject: [PATCH 051/125] docs(perf): record opaque-domain repair and remaining operand projection Refs #384, #419, #433. Retain original guards, latest-main baseline failures, ordinary 3D failure and source-domain proof limits. --- .../shared_runtime_plumbing_384_20261001.rst | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 023b75894e..f225b0b549 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -728,3 +728,79 @@ no performance or biological acceptance claim follows. Original failures remain in ``pr394-38067-graph-roi-before-20261002.log``; integrated controls are in ``pr394-graph-roi-integrated-after-20261002.log`` in RUNS. The historical PR #404 branch and its older publication proposal are not merged as a whole. + +Opaque image domain and current 3D frontier (2026-10-02) +------------------------------------------------------ + +Frozen ``45f3a721e`` diagnoses the dominant current 3D failure: scalar output +composition adds a runtime axis to an opaque whole volume. Watershed receives +``(1, 60, 128, 128)`` and fails after 118.394 seconds inside its raw callable. +The failed job peaks at 3745.977 MiB. Installed Mahotas 1.4.19's filter-offset +formula requires a 2 GiB table for that incorrect 4D seed-label neighborhood; +the singleton changes neighborhood rank without changing image voxel count. +This is a source-backed bound, not an allocator-stack capture or attribution of +the original 13.36 GB OOM. No NumBa cache files are created during the observed +pipeline window. Cache warming does not correct an invalid scientific domain. + +``d092a03af`` repairs composition using the existing nominal alignment owners. +``AlignedImageStack`` retains its declared outer runtime axis; named +``ImageOutputBundle`` members supply their original individual domains. +``PatternGroupOutputData`` derives saved member declarations from those original +values, and ``ProducedOutputSemantics`` retains that domain across physical-leaf +projection and passthrough. This execution-local fact distinguishes an opaque +single-Z volume from an explicitly declared one-plane runtime stack after both +have otherwise similar leaf metadata. It does not replace durable source ingress. +Mixed cohorts cannot manufacture one combined domain. Cache misses, overwrites, +named selection and independent pixel/mask/metadata ownership are covered. +Producer and loader bundle composition use the same existing authority. + +The first scoped original R0 rejects ten added lines in the oversized runtime +class. ``6dd47dd7e`` moves actual input assembly onto existing +``PatternGroupData`` and shared independent copying onto +``ImagePayloadStackComposition``, removing the input loader's call into the +output factory. Both unchanged original scoped R0 and R1 then pass against +``45f3a721e``; 271 controls pass, including retained lazy device resolution for +bundle composition. The whole-branch scalar relocation gate remains separate +and is not waived. Cross-device composition equivalence remains unqualified. + +A fresh ordinary CPU5/one-inline-worker execution at frozen ``d092a03af`` gets +through the first 18 steps. The entire Watershed step completes in 0.156588 +seconds. The older figure is an instrumented raw-call failure, so these are not +paired successful timings or a native speedup ratio. The full pipeline still +fails at step 18, MaskImage: a correctly selected 2D invocation receives the +opaque secondary volume without a source-proved plane projection. The refusing +2D mask guard is correct. Failure-row zero compile/execute values are wrapper +placeholders; the public compile-event span is 1.619 seconds. Pipeline clocks +exclude mandatory registry/kernel server preparation and shutdown. + +The generic relation-aware repair belongs at ``RuntimeInputBindingRequest``. +MaskImage, Crop and CorrectIlluminationApply already declare +``InputStackBroadcastSourceRelation``. That declaration permits broadcasts but +does not prove spatial-Z ordering. Contributor provenance or equal array lengths +cannot supply the missing pixel-coordinate proof. ``SourceSpatialDomain`` is +currently XY-only; source/transform ownership is being traced before extending +its existing contract. No function-specific mask path or geometry-guard +relaxation is included. Latest-head all30/native parity and ordinary performance +remain open. + +Main ``902913616`` (merged PR #434) is integrated at ``ff9d7c79d``. The combined +14-file source suite has 810 passes and two unary GrayToColor failures. Both +failures reproduce unchanged on clean main ``902913616``; they are retained, +not converted into successful assertions or treated as axis-fix regressions. +No foreign PR implementation is duplicated. + +Three completed same-server Human jobs at ``45f3a721e`` pass their scientific +comparisons and leave zero backend image-array bytes after each job. This +rejects the proposed completed-job backend leak for that workload. Generic +worker resource release already executes for inline jobs. The older OOM's full +allocation mix remains unknown; no speculative global cleanup is added. + +Receipts remain in the external benchmark RUNS directory: +``issue433-rank4-watershed-large-allocation-source-bound-20261002.json``, +``issue433-three-complete-job-retention-counterevidence-20261002.json``, +``pr394-opaque-axis-input-owner-original-r0-openhcs-20261002.json``, +``pr394-opaque-axis-input-owner-original-r1-20261002.json`` and +``d092-mask-secondary-domain-caller-audit-20261002.json``. Frozen ordinary +outputs/source hashes are retained under +``/var/tmp/openhcs-opaque-axis-d092-ordinary-20261002``. The initial failed +guard-launch arguments and the original R0 growth finding are also retained. From 6dfda95975f975f04fd2c71ddbc596dc221ed95f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 05:13:35 -0400 Subject: [PATCH 052/125] fix(runtime): publish each persisted image occurrence once (#435) --- .../core/steps/function_output_manifest.py | 22 +++ openhcs/core/steps/function_outputs.py | 41 ++++- .../unit/test_derived_role_publication_435.py | 160 ++++++++++++++++++ 3 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_derived_role_publication_435.py diff --git a/openhcs/core/steps/function_output_manifest.py b/openhcs/core/steps/function_output_manifest.py index 24ed7998dd..af9dff075f 100644 --- a/openhcs/core/steps/function_output_manifest.py +++ b/openhcs/core/steps/function_output_manifest.py @@ -144,6 +144,28 @@ def path_under(self, output_dir: str | Path) -> str: return str(Path(output_dir) / self.relative_output_path) + @property + def persisted_source_alias(self) -> str | None: + """Retain a declared artifact name on its persisted main-flow image.""" + identity = self.producer_identity + return identity.output_key if identity.artifact_kind is not None else None + + def owns_persisted_artifact( + self, + output_plan: ArtifactOutputPlan, + output_path: str, + output_dir: str | Path, + ) -> bool: + """Identify the same declared producer and exact saved occurrence.""" + identity = self.producer_identity + return ( + identity.artifact_kind is not None + and identity.artifact_kind == output_plan.artifact_type.value + and identity.output_key == output_plan.name + and identity.step_scope_id == output_plan.producer_step_scope_id + and Path(self.path_under(output_dir)) == Path(output_path) + ) + @property def output_context(self) -> AlignedImageSliceContext: """Return the declared main-flow context for this produced output.""" diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py index d8f615c97e..6b1cb7dde6 100644 --- a/openhcs/core/steps/function_outputs.py +++ b/openhcs/core/steps/function_outputs.py @@ -8,6 +8,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field, replace from pathlib import Path +from types import MappingProxyType from typing import ClassVar, TypeVar import numpy as np @@ -52,6 +53,7 @@ OpenHCSPlaneAddress, SourceArtifactProjection, SourcePlaneProjection, + SourceProjection, SourceProjectionMetadataSerializer, SourceProjectionSet, ) @@ -774,11 +776,17 @@ def runtime_artifact_projection_paths( self, context: ProcessingContext, plan: CompiledStepPlan, + *, + produced_projections: Mapping[ + Path, tuple[ProducedOutputSemantics, SourceProjection] + ] = MappingProxyType({}), ) -> tuple[tuple[SourceArtifactProjection, str], ...]: """Publish artifacts persisted in this declared storage target.""" materialization = plan.runtime_artifact_materialization if materialization.persists_to_backend(self.backend): - return self.project_runtime_artifacts(context, plan) + return self.project_runtime_artifacts( + context, plan, produced_projections=produced_projections + ) return () def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: @@ -869,6 +877,7 @@ def produced_projection_metadata( else () ) projection_paths = [] + produced_projections = {} declared_addresses: set[OpenHCSPlaneAddress] = set() parser_context = FunctionOutputParserContext.from_processing_context( context @@ -895,12 +904,16 @@ def produced_projection_metadata( SourcePlaneProjection( address=address, ref=SourcePixelRef(self.backend, virtual_path), + source_alias=record.persisted_source_alias, source_metadata=source_metadata, image_metadata=metadata, ), virtual_path, ) ) + produced_projections[Path(destination)] = ( + record, projection_paths[-1][0] + ) continue projection_paths.append( ( @@ -915,8 +928,13 @@ def produced_projection_metadata( virtual_path, ) ) + produced_projections[Path(destination)] = ( + record, projection_paths[-1][0] + ) projection_paths.extend( - self.runtime_artifact_projection_paths(context, plan) + self.runtime_artifact_projection_paths( + context, plan, produced_projections=produced_projections + ) ) if not projection_paths: return None @@ -938,6 +956,10 @@ def project_runtime_artifacts( self, context: ProcessingContext, plan: CompiledStepPlan, + *, + produced_projections: Mapping[ + Path, tuple[ProducedOutputSemantics, SourceProjection] + ] = MappingProxyType({}), ) -> tuple[tuple[SourceArtifactProjection, str], ...]: """Project persisted image artifacts into the target source authority.""" @@ -967,6 +989,21 @@ def project_runtime_artifacts( metadata ) ) + produced = produced_projections.get(Path(destination)) + if produced is not None: + record, projection = produced + if record.owns_persisted_artifact( + materialization.output_plan, destination, self.output_dir + ): + if ( + projection.address != address + or projection.image_metadata != metadata + ): + raise ValueError( + "Conflicting metadata for persisted image " + f"occurrence {destination!r}." + ) + continue source_metadata = metadata.source_component_metadata or {} persisted_source_metadata = dict(source_metadata) metadata.source_voxel_spacing.merge_into( diff --git a/tests/unit/test_derived_role_publication_435.py b/tests/unit/test_derived_role_publication_435.py new file mode 100644 index 0000000000..4a0155c0f1 --- /dev/null +++ b/tests/unit/test_derived_role_publication_435.py @@ -0,0 +1,160 @@ +"""Issue435: real saved image occurrences projected through two existing owners.""" + +from dataclasses import asdict, replace +import json + +import pytest +import numpy as np +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.memory import MemoryStorageBackend + +from test_function_outputs import ( + context_stub, function_step_plan, record_output_path, +) +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +from openhcs.constants.constants import Backend, Microscope +from openhcs.core.aligned_image_payload import AlignedImageSliceContext +from openhcs.core.artifacts import ArtifactOutputPlan, ImageArtifactType, ObjectLabelsArtifactType +from openhcs.core.compiled_step_plan import RuntimeArtifactMaterializationPlan +from openhcs.core.runtime_artifact_values import RuntimeValue +from openhcs.core.runtime_image_values import ImageMetadataPayload, ImagePayloadMetadata +from openhcs.core.runtime_stores import RuntimeValueStore +from openhcs.core.runtime_object_label_building import SourceImageObjectLabelBuildRequest +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection +from openhcs.core.steps.function_outputs import ( + PrimaryImageMetadataTarget, RuntimeArtifactMaterializationAuthority, +) +from openhcs.processing.materialization import ( + ImageFileOptions, MaterializationSpec, MaterializedFilenameIdentity, +) + + +@pytest.mark.parametrize("scenario", ("same_occurrence", "different_path", "stale_scope", "conflicting_metadata", "different_kind")) +def test_saved_roles_publish_once_per_persisted_occurrence(tmp_path, scenario): + same_persisted_path = scenario != "different_path" + filemanager = FileManager({ + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + }) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + context.microscope_handler.microscope_type = Microscope.OPENHCS.value + context.runtime_value_store = RuntimeValueStore() + context.metadata_cache = {} + context.tiff_config = None + plan = function_step_plan("Derived role images") + plan.streaming_configs = {} + plan.write_backend = Backend.DISK.value + plan.output_plate_root = str(tmp_path) + plan.output_dir = tmp_path / "images" + plan.sub_dir = "images" + plan.analysis_results_dir = str(plan.output_dir) + plan.runtime_artifact_materialization = RuntimeArtifactMaterializationPlan( + persistent_enabled=True, persistent_backend=Backend.DISK.value, + ) + context.step_plans = {plan.step_index: plan} + filemanager.ensure_directory(plan.output_dir, Backend.MEMORY.value) + components = { + "well": "A01", "site": "1", "channel": "2", + "z_index": "1", "timepoint": "1", "extension": ".tif", + } + for role, scale in (("RawRole", 1.0), ("CappedRole", 0.5)): + filename = f"A01_s001_w2_z001_t001_{role}.tif" + output_plan = ArtifactOutputPlan( + name=role, path=f"/memory/A01_s001_w2_z001_t001_{role}.pkl", + artifact_type=(ObjectLabelsArtifactType if scenario == "different_kind" and role == "CappedRole" else ImageArtifactType), + producer_step_index=plan.step_index, + producer_step_scope_id=("different-step-scope" if scenario == "stale_scope" and role == "CappedRole" else plan.step_scope_id), + producer_step_name=plan.step_name, + materialization=MaterializationSpec(ImageFileOptions( + filename_suffix=".tif", + filename_identity=MaterializedFilenameIdentity.SOURCE_IDENTITY, + relative_path_template=(filename if same_persisted_path else filename.replace(".tif", "_independent.tif")), + )), + ) + plan.artifact_outputs[output_plan.ref()] = output_plan + payload = ImageMetadataPayload( + np.ones((4, 5), dtype=np.float32) * scale, + ImagePayloadMetadata( + source_component_metadata=components, + source_image_names=(role,), + source_voxel_spacing=SourceVoxelSpacing((0.65, 0.65)), + ), + ) + artifact_payload = payload + if role == "CappedRole" and scenario == "conflicting_metadata": + artifact_payload = payload.metadata.replace_fields(source_image_names=("IndependentRole",)).payload_with(payload.data) + if role == "CappedRole" and scenario == "different_kind": + artifact_payload = SourceImageObjectLabelBuildRequest( + image=payload, labels=np.ones(payload.data.shape, dtype=np.int32), + declared_object_count=1, declared_object_ids=(1,), + ).payload() + context.runtime_value_store.record( + RuntimeValue.normalize(output_plan, artifact_payload, axis_id="A01"), + path=output_plan.path, backend=Backend.MEMORY.value, + ) + destination = plan.output_dir / filename + filemanager.save(payload, str(destination), Backend.MEMORY.value) + filemanager.ensure_directory(plan.output_dir, Backend.DISK.value) + filemanager.save(payload.data, str(destination), Backend.DISK.value) + record_output_path( + context, plan, str(destination), image_metadata=payload.metadata, + output_context=AlignedImageSliceContext.main_flow( + output_key=role, artifact_kind=ImageArtifactType.value, + ), + ) + saved = RuntimeArtifactMaterializationAuthority.materialize(context, plan) + assert len(saved) == 2 + target = replace(PrimaryImageMetadataTarget.from_plan(plan), artifact_materializations=saved) + records = target.produced_records(context, plan) + artifacts = target.runtime_artifact_projection_paths(context, plan) + receipt = { + "source_scope": plan.step_scope_id, + "produced": [{"path": str(record.path_under(plan.output_dir)), + "identity": asdict(record.producer_identity), + "address": record.filename_address.as_component_metadata()} for record in records], + "materialized": [{"path": output.path, + "role": item.materialization.output_plan.name, + "producer_scope": item.materialization.output_plan.producer_step_scope_id, + "stored_key": repr(item.materialization.record.key)} + for item in saved for output in item.outputs_for_backend(Backend.DISK.value)], + "artifact_projections": [{"path": path, + "role": projection.projection_role.value, + "address": projection.address.as_component_metadata(), + "alias": projection.source_alias, + "kind": projection.artifact_kind.value} + for projection, path in artifacts], + } + (tmp_path / "persisted-occurrence-witness.json").write_text(json.dumps(receipt, indent=2)) + produced_paths = {item["path"] for item in receipt["produced"]} + materialized_paths = {item["path"] for item in receipt["materialized"]} + if same_persisted_path: + assert produced_paths == materialized_paths + else: + assert produced_paths.isdisjoint(materialized_paths) + expected_sums = {20.0} if scenario == "different_kind" else {20.0, 10.0} + assert {float(tifffile.imread(item["path"]).sum()) for item in receipt["materialized"]} == expected_sums + if scenario != "same_occurrence": + message = "Conflicting metadata for persisted image" if scenario == "conflicting_metadata" else "Duplicate source projection address" + with pytest.raises(ValueError, match=message) as rejection: + target.produced_projection_metadata(context, plan) + receipt["rejection"] = str(rejection.value) + (tmp_path / "persisted-occurrence-witness.json").write_text(json.dumps(receipt, indent=2)) + return + structured = target.produced_projection_metadata(context, plan) + assert len(structured["source_projection"]) == 2 + assert {item["source_alias"] for item in structured["source_projection"]} == {"RawRole", "CappedRole"} + target.write(context, produced_plan=plan) + document = json.loads((tmp_path / "openhcs_metadata.json").read_text()) + reopened = VirtualWorkspaceSourceProjection.from_openhcs_metadata(tmp_path, document) + projections = reopened.source_projections_by_virtual_path + relative_paths = {item["path"].removeprefix(str(tmp_path) + "/") for item in receipt["produced"]} + assert set(projections) == relative_paths | produced_paths + assert {projection.source_alias for projection in projections.values()} == {"RawRole", "CappedRole"} + for relative_path in relative_paths: + projection = projections[relative_path] + assert projection.address.as_component_metadata() == {key: components[key] for key in ("well", "site", "channel", "z_index", "timepoint")} + expected_scale = 1.0 if projection.source_alias == "RawRole" else 0.5 + np.testing.assert_array_equal(tifffile.imread(tmp_path / relative_path), np.ones((4, 5), dtype=np.float32) * expected_scale) From ce1b7acfc086e16044649fbfb67b2d244e703496 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 05:25:39 -0400 Subject: [PATCH 053/125] refactor(runtime): derive saved alias from output context (#435) --- openhcs/core/aligned_image_payload.py | 5 +++++ .../core/steps/function_output_manifest.py | 20 +++++++++---------- openhcs/core/steps/function_outputs.py | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/openhcs/core/aligned_image_payload.py b/openhcs/core/aligned_image_payload.py index 98451b0e19..ac312ba295 100644 --- a/openhcs/core/aligned_image_payload.py +++ b/openhcs/core/aligned_image_payload.py @@ -1012,6 +1012,11 @@ class AlignedImageSliceContext: projection_key: str artifact_kind: str | None = None + @property + def persisted_source_alias(self) -> str | None: + """Expose a declared artifact name, preserving anonymous main flow.""" + return self.output_key if self.artifact_kind is not None else None + @classmethod def main_flow( cls, diff --git a/openhcs/core/steps/function_output_manifest.py b/openhcs/core/steps/function_output_manifest.py index af9dff075f..daef36dea2 100644 --- a/openhcs/core/steps/function_output_manifest.py +++ b/openhcs/core/steps/function_output_manifest.py @@ -144,12 +144,6 @@ def path_under(self, output_dir: str | Path) -> str: return str(Path(output_dir) / self.relative_output_path) - @property - def persisted_source_alias(self) -> str | None: - """Retain a declared artifact name on its persisted main-flow image.""" - identity = self.producer_identity - return identity.output_key if identity.artifact_kind is not None else None - def owns_persisted_artifact( self, output_plan: ArtifactOutputPlan, @@ -159,11 +153,15 @@ def owns_persisted_artifact( """Identify the same declared producer and exact saved occurrence.""" identity = self.producer_identity return ( - identity.artifact_kind is not None - and identity.artifact_kind == output_plan.artifact_type.value - and identity.output_key == output_plan.name - and identity.step_scope_id == output_plan.producer_step_scope_id - and Path(self.path_under(output_dir)) == Path(output_path) + self.output_context.persisted_source_alias, + identity.artifact_kind, + identity.step_scope_id, + Path(self.path_under(output_dir)), + ) == ( + output_plan.name, + output_plan.artifact_type.value, + output_plan.producer_step_scope_id, + Path(output_path), ) @property diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py index 6b1cb7dde6..fcaef7bf7f 100644 --- a/openhcs/core/steps/function_outputs.py +++ b/openhcs/core/steps/function_outputs.py @@ -904,7 +904,7 @@ def produced_projection_metadata( SourcePlaneProjection( address=address, ref=SourcePixelRef(self.backend, virtual_path), - source_alias=record.persisted_source_alias, + source_alias=record.output_context.persisted_source_alias, source_metadata=source_metadata, image_metadata=metadata, ), From 919fee286f8003a00aafc023ad28767d9713842b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 05:28:59 -0400 Subject: [PATCH 054/125] docs(runtime): qualify derived-role publication repair (#435) --- .../shared_runtime_plumbing_384_20261001.rst | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index f225b0b549..dc29a52481 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -804,3 +804,47 @@ Receipts remain in the external benchmark RUNS directory: outputs/source hashes are retained under ``/var/tmp/openhcs-opaque-axis-d092-ordinary-20261002``. The initial failed guard-launch arguments and the original R0 growth finding are also retained. + +Persisted derived-role publication (issue #435) +---------------------------------------------- + +``ce1b7acfc`` repairs duplicate publication of one saved image occurrence in the +existing metadata writer. The existing output context derives the declared +source alias; anonymous main flow remains anonymous. Produced records and +successful materialization outcomes share a path index. A second projection is +omitted only when declared alias, artifact kind, producer scope, exact path, +scalar acquisition address and full persisted image metadata agree. Distinct +files with colliding identities, stale scopes, different kinds and conflicting +metadata retain their original refusal. No source-projection guard is relaxed. +The index is linear in produced records and successful outputs. + +The actual writer reproducer at baseline ``130c03a26`` saves two distinct role +images with real C2 addresses and calibration, then reproduces the duplicate +projection failure. At ``ce1b7acfc``, both roles publish once and reopen through +the existing virtual workspace; exact pixels, addresses and spacing pass. All +four independent refusal controls pass, together with 232 focused controls in +9.42 seconds. These are writer/materialization/reopening controls, not an +installed five-step acceptance or a performance benchmark. + +Unchanged original scoped R0 and R1 pass for ``130c03a26`` to ``ce1b7acfc``. +The first candidate ``6dfda9597`` failed original R0 for a long producer-match +Boolean and a foreign absence probe. Moving alias derivation to its existing +context owner and comparing exact producer tuples resolves those findings. +The whole-branch scalar gate remains separate. Independent review finds no +blocker; persisted occurrence identity does not prove stale in-memory pixels +equal the final overwritten file. Pixel parity remains a separate gate. + +The parent has supplied the original five-step source and bounded saved-path +facts in issue comment ``5948666249``. Actual produced-record and materialized +output pairs were not serialized there. Installed original-path acceptance +remains open; no biological replay or fabricated collision pair is claimed. +The separate #437 owner retains its singleton-consumption repair. + +Main ``8551a4864`` (merged #438 and #439) is integrated at ``3de415d63``. +The integrated publication, axis and new MCP compilation controls pass: +247 tests in 51.50 seconds, with two existing pytest configuration warnings. +The scoped guard receipts are retained externally as +``issue435-ce1-original-r0-retry-20261002.json`` and +``issue435-ce1-original-r1-20261002.json``; independent review is +``/var/tmp/openhcs-issue435-independent-publication-review-20261002.json``. +Original failures and failed guard launches remain retained. From a5134a0b85182078700ac06f27dffd3e5788302b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 06:26:30 -0400 Subject: [PATCH 055/125] docs(perf): record fresh CP comparison and measurement frontier (#384, #444) --- .../shared_runtime_plumbing_384_20261001.rst | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index dc29a52481..8aaa9949d8 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -848,3 +848,69 @@ The scoped guard receipts are retained externally as ``issue435-ce1-original-r1-20261002.json``; independent review is ``/var/tmp/openhcs-issue435-independent-publication-review-20261002.json``. Original failures and failed guard launches remain retained. + +Fresh native comparison and measurement lifetime frontier (2026-10-02) +-------------------------------------------------------------------- + +Frozen ``919fee286f8003a00aafc023ad28767d9713842b`` completed two ordinary +public-driver observations on CPU5, one inline worker/thread, with default +OUTCOMES and the default memory observer. Server startup, mandatory registry +and kernel warmup, and shutdown are excluded from pipeline clocks. + +===================== =========== ============== ============== ============== +Case Compilation Server job Pipeline total Native CP run +===================== =========== ============== ============== ============== +Advanced segmentation 1.324832 s 8.078740 s 10.650795 s 34.327521 s +Imaging flow cytometry 1.941105 s 15.497744 s 18.177965 s 77.795280 s +===================== =========== ============== ============== ============== + +Native CellProfiler 4.2.8.1 uses the same physical input paths and SHA256 +values, CPU5 and one thread. Its invocation excludes interpreter/JVM startup, +pipeline load and one complete warmup, but includes image-set preparation, +modules, post_run and Measurements.close. These are single observations with +different explicit setup/publication accounting; they do not establish a +kernel ratio, statistical significance or a new patch's speedup. + +IFC's existing strict exported-science comparison passes with zero differences, +1800 nonempty rows and 527 columns in each CSV. The observed native/OH ratios +are 5.019781 for server-job execution and 4.279647 for pipeline total. Unsaved +segmentation masks are outside these retained outputs. Advanced's existing +scientific database comparison also passes with zero differences, but complete +source-information acceptance remains RED: five illumination sources across +two sites have 90 NULL cells instead of native values. The original integer +gate detects 40 Frame/Series/Height/Width cells; independent inventory adds +ten Scaling and forty FileName/PathName/URL/MD5Digest cells. Issue #444 is +formally linked to PR #394. No field exclusion or tolerance change is admitted. + +A separate coarse diagnostic on the same source partitions IFC's 13.838053 s +execution into 9.373829 s inside the raw-callable boundary and 4.464224 s +outside. Advanced's 8.087572 s partitions into 3.836250 s inside and 4.251322 s +outside. The callable boundary includes decorated processing and result +assembly, not just numerical kernels. Nested inclusive spans are not added. +Exclusive recording, table assembly and export work has a combined envelope +of 2.812474 s for IFC and 2.882479 s for Advanced. Saving two seconds requires +roughly 71.11% and 69.38% reduction respectively; recording alone is insufficient. +Stack loading costs 0.126811 s / 0.106307 s. Ordinary progress events bound +between-step/axis work below 17 ms and other job-minus-axis gaps at +0.196432 s / 0.100492 s. Those routes cannot close the multi-second target. +Diagnostic clocks are not ordinary speedup evidence. + +Retained evidence lives under ``/var/tmp``: ordinary outputs and source freeze +in ``openhcs-latest-ordinary-advanced-ifc-20261002``, fresh native inputs, +reports and IFC comparison in +``openhcs-current-native-advanced-ifc-v2-919fee-20261002``, and coarse regions +in ``openhcs-current919-runtime-phase-probe-20261002``. Independent source-field +inventory is ``advanced-fresh-native-missing-illum-metadata-20261002.json``; +the dominant cost/owner receipt is +``openhcs-919-measurement-lifetime-frontier-20261002.json``. Original failed +native preparation and the exact metadata comparison failure remain retained. +Actual typed measurement-request/export capture is in progress; no production +optimization or replay acceptance follows from the cost envelope alone. + +Latest main ``3df650bf7`` (merged #442/#443) is integrated at ``436e49755``. +The incoming BioFormats calibration, MCP streaming and existing publication/ +opaque-domain controls pass: 141 tests in 16.72 seconds. The initial narrow +launch failed collection because the MCP test imports the streaming fixture +module by its short name; the corrected command includes that actual fixture +owner first. Both logs are retained in ``/var/tmp``. No source/test guard was +modified to obtain the pass. From 8f994c81a9aa7f2c87f29dda5aec2316bc1dacd3 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 06:45:02 -0400 Subject: [PATCH 056/125] docs(perf): retain failed recording premise and qualify export replay (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 8aaa9949d8..29acca111b 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -871,8 +871,12 @@ modules, post_run and Measurements.close. These are single observations with different explicit setup/publication accounting; they do not establish a kernel ratio, statistical significance or a new patch's speedup. -IFC's existing strict exported-science comparison passes with zero differences, -1800 nonempty rows and 527 columns in each CSV. The observed native/OH ratios +IFC's existing strict exported-science comparison passes with zero differences +and 1800 nonempty rows in each CSV. Native has 528 physical columns and OH 527: +the native redundant Number_Object_Number equals ObjectNumber for all 1800 +rows, and the existing nominal identity policy recognizes that declaration. +This is semantic measurement equality, not identical cross-tool CSV headers. +The observed native/OH ratios are 5.019781 for server-job execution and 4.279647 for pipeline total. Unsaved segmentation masks are outside these retained outputs. Advanced's existing scientific database comparison also passes with zero differences, but complete @@ -914,3 +918,30 @@ launch failed collection because the MCP test imports the streaming fixture module by its short name; the corrected command includes that actual fixture owner first. Both logs are retained in ``/var/tmp``. No source/test guard was modified to obtain the pass. + +The actual IFC capture completed the public pipeline and exact source/dependency +freeze, but its four-snapshot admission remains RED: step22 never invokes +MeasurementsOutputRecorder.record. Only the terminal spreadsheet before/after +graphs were captured (44,935,114 and 56,109,057 bytes), with no graph or budget +rejection. The nine real recorder calls publish identification/filtering/ +expansion/relationship core tables. Texture and intensity-distribution tables +use the separate per-object recorder. Thus the proposed capture and the premise +that all outer recording cost is redundant wide-feature conversion are rejected. + +Independent export-only qualification passes both unchanged V6 loads (615 arrays +each), exact original CSV bytes, existing strict CP science, and all determining +field/source/array aliases. Thirty-five immutable metadata owners drop their +derived views by their existing declared transport policy; every populated view +matches fresh recomputation from the same fields. Original raw-heap equality +remains RED for that intentionally untransported cache state, separately from +declared-state equality. The actual exporter graph has 31 measurement tables +and two relationship records, with no label pixels; it cannot validate a dense +label-storage optimization. Local diagnostic profiling finds repeated row +identity/axis scanning and assignment; it is not end-to-end speedup evidence. +The ordinary export ceiling is 1.4077 seconds, so export alone cannot save two +seconds. Preserving the native CSV span leaves an optimistic combined outer +recording/export ceiling of 2.4189 seconds; a two-second saving requires about +82.68% collapse. No implementation is admitted by that ceiling alone. +Receipts are ``openhcs-measurement-export-declared-state-profile-20261002`` and +``openhcs-measurement-export-declared-fields-alias-control-20261002.json`` under +``/var/tmp``. The original four-snapshot and raw-heap failures remain retained. From dd22e1bf98c38f4bd04e4a8d6cb934f0c8c93681 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 06:44:21 -0400 Subject: [PATCH 057/125] fix: export declared secondary image source occurrences Collect source-bound CPA image metadata independently of recorded producer outputs, while retaining executed image-set correlation, exact source occurrence scope, and conflicting-record refusal. Read TIFF and NumPy header geometry through the existing file-format metadata owner and revision cache, including explicit TIFF frame selection. Refs #444; production Advanced export parity remains pending. --- openhcs/core/image_file_serialization.py | 85 ++++++- openhcs/core/source_projection.py | 5 + openhcs/core/source_workspace_projection.py | 22 ++ .../interop/cellprofiler/analyst_export.py | 150 +++++++++--- .../cellprofiler/image_set_numbering.py | 36 ++- .../unit/test_cellprofiler_analyst_export.py | 231 ++++++++++++++++++ tests/unit/test_image_file_serialization.py | 68 ++++++ 7 files changed, 551 insertions(+), 46 deletions(-) diff --git a/openhcs/core/image_file_serialization.py b/openhcs/core/image_file_serialization.py index a83b2b3bbc..4ab05b2b94 100644 --- a/openhcs/core/image_file_serialization.py +++ b/openhcs/core/image_file_serialization.py @@ -69,6 +69,62 @@ class ImageFileSourceMetadata: source_dtype: Any | None = None intensity_scale: float | None = None pixel_semantics: SourceImagePixelSemantics = SourceImagePixelSemantics() + image_shape_yx: tuple[int, int] | None = None + source_frame_shape: tuple[int, ...] | None = () + + def __post_init__(self) -> None: + if self.image_shape_yx is not None: + if len(self.image_shape_yx) != 2 or any( + not isinstance(value, int) or isinstance(value, bool) or value <= 0 + for value in self.image_shape_yx + ): + raise ValueError( + "Image-file YX shape requires two positive integer dimensions." + ) + object.__setattr__(self, "image_shape_yx", tuple(self.image_shape_yx)) + if self.source_frame_shape is not None: + if any( + not isinstance(value, int) or isinstance(value, bool) or value <= 0 + for value in self.source_frame_shape + ): + raise ValueError( + "Image-file frame shape requires positive integer dimensions." + ) + object.__setattr__( + self, "source_frame_shape", tuple(self.source_frame_shape) + ) + + def frame_for_source_indices(self, indices: tuple[int, ...]) -> int: + """Map explicitly selected leading source axes to a container frame.""" + + if not indices: + return 0 + if self.source_frame_shape is None or len(indices) != len( + self.source_frame_shape + ): + raise ValueError( + "Source selection has no declared complete container-frame mapping." + ) + frame = 0 + for index, size in zip(indices, self.source_frame_shape, strict=True): + if ( + not isinstance(index, int) + or isinstance(index, bool) + or not 0 <= index < size + ): + raise ValueError( + f"Source frame index {index!r} is outside dimension {size}." + ) + frame = frame * size + index + return frame + + def require_image_geometry(self) -> tuple[Any, int, int]: + """Require complete format-declared dtype and physical image dimensions.""" + + if self.image_shape_yx is None or self.source_dtype is None: + raise ValueError("Image-file header does not declare image dtype and YX geometry.") + height, width = self.image_shape_yx + return self.source_dtype, height, width def project_image_metadata( self, metadata: ImagePayloadMetadata, *, values_preserved: bool @@ -219,14 +275,23 @@ def _read_source_metadata(cls, path: Path) -> ImageFileSourceMetadata: import imageio.v3 as iio - dtype = iio.improps(path).dtype + properties = iio.improps(path) + dtype = properties.dtype + pixel_semantics = cls.require_pixel_semantics(path) + shape = tuple(properties.shape) + if pixel_semantics.channel_axis is not None: + axis = pixel_semantics.channel_axis % len(shape) + shape = shape[:axis] + shape[axis + 1 :] return ImageFileSourceMetadata( source_dtype=dtype, intensity_scale=( cls.declared_intensity_scale(path) or image_intensity_scale_for_dtype(dtype) ), - pixel_semantics=cls.require_pixel_semantics(path), + pixel_semantics=pixel_semantics, + image_shape_yx=( + tuple(int(value) for value in shape) if len(shape) == 2 else None + ), ) def preserves_pixel_values(self, source_dtype: Any) -> bool: @@ -313,6 +378,8 @@ def _read_source_metadata(cls, path: Path) -> ImageFileSourceMetadata: return ImageFileSourceMetadata( source_dtype=array.dtype, intensity_scale=image_intensity_scale_for_dtype(array.dtype), + image_shape_yx=tuple(array.shape) if array.ndim == 2 else None, + source_frame_shape=() if array.ndim == 2 else None, ) @@ -346,6 +413,18 @@ def _read_source_metadata(cls, path: Path) -> ImageFileSourceMetadata: series = tif.series[0] dtype = series.dtype declared_scale = cls._declared_intensity_scale_from_page(tif.pages[0]) + image_shape_yx = ( + int(tif.pages[0].imagelength), + int(tif.pages[0].imagewidth), + ) + spatial_axis = series.axes.find("Y") + frame_shape = tuple(int(value) for value in series.shape[:spatial_axis]) + if ( + spatial_axis < 0 + or "S" in series.axes[:spatial_axis] + or len(series.pages) != np.prod(frame_shape) + ): + frame_shape = None sample_axis = series.axes.find("S") sample_count = int(series.shape[sample_axis]) if sample_axis >= 0 else None pixel_semantics = SourceImagePixelSemantics() @@ -360,6 +439,8 @@ def _read_source_metadata(cls, path: Path) -> ImageFileSourceMetadata: source_dtype=dtype, intensity_scale=(declared_scale or image_intensity_scale_for_dtype(dtype)), pixel_semantics=pixel_semantics, + image_shape_yx=image_shape_yx, + source_frame_shape=frame_shape, ) @classmethod diff --git a/openhcs/core/source_projection.py b/openhcs/core/source_projection.py index 38bf869dcd..8711e50c62 100644 --- a/openhcs/core/source_projection.py +++ b/openhcs/core/source_projection.py @@ -707,6 +707,11 @@ def matches_binding(self, binding: NamedSourceBinding) -> bool: and self.artifact_kind is binding.artifact_kind ) + def belongs_to_execution_axis(self, axis_id: str) -> bool: + """Retain an unscoped source or its exact declared execution axis.""" + + return self.execution_scope is None or self.execution_scope.axis_id == axis_id + def declared_optional_payload_field(cls, payload_type: type) -> str: """Derive the one declared field carrying ``payload_type | None``. diff --git a/openhcs/core/source_workspace_projection.py b/openhcs/core/source_workspace_projection.py index 8caef9e15f..0b3216e025 100644 --- a/openhcs/core/source_workspace_projection.py +++ b/openhcs/core/source_workspace_projection.py @@ -21,6 +21,7 @@ from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.source_bindings import ( SOURCE_BINDING_ALIAS_METADATA_FIELD, + NamedSourceBinding, SourceProjectionRole, ) from openhcs.core.source_metadata import ( @@ -370,6 +371,27 @@ def files_for_projection_role( is role ) + def source_occurrences_for_binding( + self, + binding: NamedSourceBinding, + *, + axis_id: str, + ) -> tuple[tuple[str, SourceProjection], ...]: + """Select exact declared occurrences without merging physical resources.""" + + occurrences = [] + for path in self.files_for_projection_role( + binding.projection_role, axis_id=axis_id + ): + projection = self.require_source_projection_for( + VirtualWorkspacePathLookup.from_paths(path, path) + ) + if projection.matches_binding( + binding + ) and projection.belongs_to_execution_axis(axis_id): + occurrences.append((path, projection)) + return tuple(occurrences) + def validate_runtime_metadata_projection( self, *, diff --git a/openhcs/interop/cellprofiler/analyst_export.py b/openhcs/interop/cellprofiler/analyst_export.py index 4d1df75243..8aeacc0359 100644 --- a/openhcs/interop/cellprofiler/analyst_export.py +++ b/openhcs/interop/cellprofiler/analyst_export.py @@ -30,6 +30,7 @@ ) from openhcs.core.component_group_scope import RuntimeExecutionAxisScope from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.image_file_serialization import ImageFileFormat from openhcs.core.measurement_row_materialization import ( WideMeasurementRowAccumulator, ) @@ -185,10 +186,13 @@ def _source_image_projection_values( source_path: Path, source_image_name: str, dialect: CellProfilerDatabaseColumnDialect, + *, + source_axis_indices: tuple[int, ...] = (), ) -> Mapping[str, Any]: - with Image.open(source_path) as source_image: - width, height = source_image.size - source_dtype = np.asarray(source_image).dtype + header = ImageFileFormat.require_path(source_path).require_source_metadata( + source_path + ) + source_dtype, height, width = header.require_image_geometry() if np.issubdtype(source_dtype, np.integer): scaling = float(np.iinfo(source_dtype).max) elif np.issubdtype(source_dtype, np.bool_): @@ -196,7 +200,9 @@ def _source_image_projection_values( else: scaling = 1.0 values_by_field = { - CellProfilerSourceImageProjectionField.FRAME: 0, + CellProfilerSourceImageProjectionField.FRAME: header.frame_for_source_indices( + source_axis_indices + ), CellProfilerSourceImageProjectionField.HEIGHT: height, CellProfilerSourceImageProjectionField.MD5_DIGEST: md5( source_path.read_bytes(), @@ -877,6 +883,7 @@ def collect_image_provenance( ], thumbnail_field: FieldSpec | None = None, auto_scale_thumbnail_intensities: bool = True, + source_axis_indices: tuple[int, ...] = (), ) -> None: """Fold typed source provenance directly into projected image rows.""" @@ -899,11 +906,13 @@ def collect_image_provenance( source_path, source_image_name, self.dialect, + source_axis_indices=source_axis_indices, ) ) if thumbnail_field is not None: - with Image.open(source_path) as source_image: - pixels = np.asarray(source_image) + pixels = ImageFileFormat.require_path(source_path).read( + source_path + ) values[thumbnail_field.name] = _thumbnail_png_base64( pixels, auto_scale=auto_scale_thumbnail_intensities, @@ -939,6 +948,97 @@ def collect_image_provenance( ) source_metadata_by_image_number[image_number].append(metadata_items) + def collect_recorded_image_provenance( + self, + *, + records: Sequence[StoredRuntimeValue], + image_channels: Sequence[CPAImageChannelSpec], + image_rows_by_number: dict[int, dict[str, Any]], + source_metadata_by_image_number: dict[ + int, + list[Mapping[str, SourceMetadataScalar]], + ], + ) -> None: + for channel in image_channels: + channel_records = tuple( + record for record in records if record.key.name == channel.alias + ) + for record in channel_records: + provenance = image_payload_metadata(record.value.data).source_provenance + self.collect_image_provenance( + provenance, + scope=record.key.scope, + source_image_name=channel.alias, + image_rows_by_number=image_rows_by_number, + source_metadata_by_image_number=source_metadata_by_image_number, + ) + + def collect_source_bound_image_provenance( + self, + *, + source_binding_plan: CompiledSourceBindingPlan, + image_channels: Sequence[CPAImageChannelSpec], + axis_id: str, + image_rows_by_number: dict[int, dict[str, Any]], + source_metadata_by_image_number: dict[ + int, list[Mapping[str, SourceMetadataScalar]] + ], + ) -> None: + """Project declared source occurrences for image sets actually exported.""" + + if self.context is None: + return + workspace = VirtualWorkspaceSourceProjectionAuthority.from_context( + self.context, + cache=self.context.runtime_source_workspace_projection_cache, + ).projection_if_available() + if workspace is None: + return + channel_aliases = frozenset(channel.alias for channel in image_channels) + scope = RuntimeExecutionAxisScope(axis_id) + for binding in source_binding_plan.binding_declarations: + if ( + binding.alias not in channel_aliases + or binding.artifact_kind is not ImageArtifactType + ): + continue + for path, projection in workspace.source_occurrences_for_binding( + binding, axis_id=axis_id + ): + lookup = VirtualWorkspacePathLookup.from_paths(path, path) + provenance = SourceImageProvenance( + source_path=path, + source_component_metadata=workspace.source_metadata_for(lookup), + source_image_names=(binding.alias,), + ) + if ( + self.image_set_numbering.existing_number_for_source_slice( + scope=scope, + provenance=provenance, + slice_index=0, + owner=binding.alias, + ) + is None + ): + continue + selection = binding.explicit_source + if selection is not None and ( + selection.series not in (None, "0") + or selection.index not in (None, "0") + ): + raise ValueError( + f"CPA source image {binding.alias!r} requires format-owned " + "Frame/Series projection for its embedded source selection." + ) + self.collect_image_provenance( + provenance, + scope=scope, + source_image_name=binding.alias, + image_rows_by_number=image_rows_by_number, + source_metadata_by_image_number=source_metadata_by_image_number, + source_axis_indices=projection.ref.source_axis_indices, + ) + def image_numbers_for_provenance( self, provenance: SourceImageProvenance, @@ -1128,11 +1228,16 @@ def build( image_rows_by_number=image_rows_by_number, source_metadata_by_image_number=source_metadata_by_image_number, ) - self._collect_image_provenance( + row_projection.collect_recorded_image_provenance( records=image_records[axis_id], image_channels=image_channels, - settings=settings, - row_projection=row_projection, + image_rows_by_number=image_rows_by_number, + source_metadata_by_image_number=source_metadata_by_image_number, + ) + row_projection.collect_source_bound_image_provenance( + source_binding_plan=self.source_binding_plan, + image_channels=image_channels, + axis_id=axis_id, image_rows_by_number=image_rows_by_number, source_metadata_by_image_number=source_metadata_by_image_number, ) @@ -1444,33 +1549,6 @@ def _collect_relationships( rows = row_projection.relationship_rows(record, relationship) relationship_rows_by_name[relationship.name].extend(rows) - @staticmethod - def _collect_image_provenance( - *, - records: Sequence[StoredRuntimeValue], - image_channels: Sequence[CPAImageChannelSpec], - settings: CellProfilerDatabaseExportSettings, - row_projection: CPATableRowProjection, - image_rows_by_number: dict[int, dict[str, Any]], - source_metadata_by_image_number: dict[ - int, - list[Mapping[str, SourceMetadataScalar]], - ], - ) -> None: - for channel in image_channels: - channel_records = tuple( - record for record in records if record.key.name == channel.alias - ) - for record in channel_records: - provenance = image_payload_metadata(record.value.data).source_provenance - row_projection.collect_image_provenance( - provenance, - scope=record.key.scope, - source_image_name=channel.alias, - image_rows_by_number=image_rows_by_number, - source_metadata_by_image_number=source_metadata_by_image_number, - ) - @staticmethod def _collect_measurement_provenance( *, diff --git a/openhcs/interop/cellprofiler/image_set_numbering.py b/openhcs/interop/cellprofiler/image_set_numbering.py index ac119612ef..f3c9a8f3c1 100644 --- a/openhcs/interop/cellprofiler/image_set_numbering.py +++ b/openhcs/interop/cellprofiler/image_set_numbering.py @@ -43,19 +43,39 @@ def for_source_slices( return { slice_index: self._numbers.setdefault( - ( - scope.axis_id, - self._source_identity( - provenance, - slice_index, - owner=owner, - ), - ), + self._source_key(scope, provenance, slice_index, owner=owner), len(self._numbers) + 1, ) for slice_index in slice_indices } + def existing_number_for_source_slice( + self, + *, + scope: RuntimeExecutionAxisScope, + provenance: SourceImageProvenance, + slice_index: int, + owner: str, + ) -> int | None: + """Find an executed image set without admitting an unused source occurrence.""" + + return self._numbers.get( + self._source_key(scope, provenance, slice_index, owner=owner) + ) + + def _source_key( + self, + scope: RuntimeExecutionAxisScope, + provenance: SourceImageProvenance, + slice_index: int, + *, + owner: str, + ) -> tuple[str, SourceImageSetIdentity]: + return ( + scope.axis_id, + self._source_identity(provenance, slice_index, owner=owner), + ) + def for_source_slice( self, *, diff --git a/tests/unit/test_cellprofiler_analyst_export.py b/tests/unit/test_cellprofiler_analyst_export.py index 89390f3370..7c2ee83497 100644 --- a/tests/unit/test_cellprofiler_analyst_export.py +++ b/tests/unit/test_cellprofiler_analyst_export.py @@ -67,6 +67,15 @@ SourceImageIdentity, SourceImageProvenanceContributor, ) +from openhcs.core.source_projection import ( + OpenHCSPlaneAddress, + SourceArtifactProjection, + SourcePlaneProjection, + SourceProjectionMetadataSerializer, + SourceProjectionSet, +) +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +from polystore.virtual_workspace import SourcePixelRef from openhcs.interop.cellprofiler.analyst_export import ( CPAImageChannelSpec, CPAPropertiesRenderer, @@ -682,6 +691,228 @@ def test_projection_renders_source_builtins_and_thumbnail_from_declared_image( } <= {field_spec.name for field_spec in projection.image_table.columns} +def _borrowed_source_export_fixture( + tmp_path: Path, + *, + embedded_tiff: bool = False, + calibration_scope=None, +): + """Real source workspace occurrences and measurements, without image records.""" + calibration = tmp_path / ("IllumDNA.tif" if embedded_tiff else "IllumDNA.npy") + if embedded_tiff: + import tifffile + + tifffile.imwrite( + calibration, + np.ones((3, 5, 7), dtype=np.uint16), + photometric="minisblack", + metadata={"axes": "ZYX"}, + ) + else: + np.save(calibration, np.ones((5, 7), dtype=np.float32)) + sources = [] + source_metadata = [] + for site in (1, 2, 3): + original = tmp_path / f"dna-{site}.tif" + Image.fromarray(np.zeros((5, 7), dtype=np.uint16)).save(original) + metadata = {"well": "A01", "site": str(site), "z_index": "1", "timepoint": "1"} + source_metadata.append(metadata) + sources.extend( + ( + SourcePlaneProjection( + address=OpenHCSPlaneAddress.from_values("A01", site, 1, 1, 1), + ref=SourcePixelRef("disk", str(original)), + source_alias="DNA", + source_metadata=metadata, + ), + SourceArtifactProjection( + address=OpenHCSPlaneAddress.from_values("A01", site, 2, 1, 1), + ref=SourcePixelRef( + "disk", str(calibration), (site - 1,) if embedded_tiff else () + ), + source_alias="IllumDNA", + artifact_kind=ImageArtifactType, + source_metadata=metadata, + execution_scope=calibration_scope, + ), + ) + ) + serializer = SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()) + document = { + "subdirectories": { + "source": serializer.metadata_dict( + SourceProjectionSet(tuple(sources)), + microscope_handler_name="openhcs", + source_filename_parser_name="source_schema", + grid_dimensions=[1, 1], + pixel_size=1.0, + ) + } + } + context = _export_context() + context.plate_path = tmp_path + context.microscope_handler = SimpleNamespace( + metadata_handler=SimpleNamespace( + source_workspace_metadata_document=lambda _plate_path: document, + ) + ) + context.filemanager = SimpleNamespace( + exists=lambda *_args: False, + source_path=lambda address, backend, *, base_path: address, + ) + plan = CompiledSourceBindingPlan( + bindings=( + NamedSourceBinding(alias="DNA"), + NamedSourceBinding( + alias="IllumDNA", projection_role=SourceProjectionRole.SOURCE_ARTIFACT + ), + ) + ) + table = MeasurementTable( + name="ImageMeasurements", + rows=MeasurementSparseColumnarRows.from_rows( + tuple( + {RUNTIME_IMAGE_FIELD: index, "Count_Nuclei": index + 1} + for index in range(2) + ), + fields=( + FieldSpec(RUNTIME_IMAGE_FIELD, int), + FieldSpec("Count_Nuclei", int), + ), + ), + subject=MeasurementSubject(MeasurementScope.IMAGE, "Image"), + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + component_metadata=tuple(source_metadata[:2]), + ), + ) + store = RuntimeValueStore() + output = ArtifactOutputPlan( + name=table.name, + path="/memory/measurements.pkl", + artifact_type=MeasurementsArtifactType, + ) + store.record( + RuntimeValue.normalize(output, table, axis_id="A01"), + path=output.path, + backend="memory", + ) + batch = RuntimeArtifactBatch( + input_specs=( + ArtifactSpec.input("DNA", ImageArtifactType), + ArtifactSpec.input("IllumDNA", ImageArtifactType), + ArtifactSpec.input(table.name, MeasurementsArtifactType), + ), + records_by_axis={"A01": store.values()}, + source_image_set_identity_policy=SourceImageSetIdentityPolicy( + frozenset((AllComponents.CHANNEL,)) + ), + source_binding_plan=plan, + ) + builder = CellProfilerAnalystProjectionBuilder(plan, context=context) + channels = CPAImageChannelSpec.defaults_for_artifacts( + batch.specs_of_type(ImageArtifactType), source_binding_plan=plan + ) + return builder, batch, channels, calibration, document + + +def test_source_only_calibration_occurrences_export_each_executed_site( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + builder, batch, channels, calibration, _document = _borrowed_source_export_fixture( + tmp_path + ) + assert batch.records_of_type(ImageArtifactType)["A01"] == () + monkeypatch.setattr( + np, + "asarray", + lambda *_args, **_kwargs: pytest.fail( + "Source headers must not decode pixel arrays" + ), + ) + + projection = builder.build(batch, _settings(), channels) + + rows = _external_rows(projection.image_table) + assert len(rows) == 2 # The third declared source site was never executed. + for image_number, row in enumerate(rows, start=1): + assert row["ImageNumber"] == image_number + assert row["Image_FileName_IllumDNA"] == calibration.name + assert row["Image_PathName_IllumDNA"] == str(tmp_path) + assert row["Image_URL_IllumDNA"] == calibration.as_uri() + assert row["Image_Height_IllumDNA"] == 5 + assert row["Image_Width_IllumDNA"] == 7 + assert row["Image_Frame_IllumDNA"] == row["Image_Series_IllumDNA"] == 0 + assert row["Image_Scaling_IllumDNA"] == 1.0 + assert len(row["Image_MD5Digest_IllumDNA"]) == 32 + assert row["Image_FileName_DNA"] == f"dna-{image_number}.tif" + assert row["Image_Scaling_DNA"] == 65535.0 + + +def test_source_bound_export_refuses_conflicting_recorded_alias(tmp_path: Path) -> None: + builder, batch, channels, _calibration, _document = _borrowed_source_export_fixture( + tmp_path + ) + other = tmp_path / "different.npy" + np.save(other, np.ones((5, 7), dtype=np.float32)) + store = RuntimeValueStore() + output = ArtifactOutputPlan( + name="IllumDNA", path="/memory/conflict.pkl", artifact_type=ImageArtifactType + ) + payload = ImagePayloadMetadata( + source_path=str(other), + source_component_metadata={ + "well": "A01", + "site": "1", + "z_index": "1", + "timepoint": "1", + }, + ).attach_to(np.ones((5, 7), dtype=np.float32)) + store.record( + RuntimeValue.normalize(output, payload, axis_id="A01"), + path=output.path, + backend="memory", + ) + batch = replace( + batch, records_by_axis={"A01": (*batch.records_by_axis["A01"], *store.values())} + ) + with pytest.raises(ValueError, match="conflicting values.*Image_FileName_IllumDNA"): + builder.build(batch, _settings(), channels) + + +def test_source_bound_export_retains_exact_embedded_tiff_frame_selection( + tmp_path: Path, +) -> None: + builder, batch, channels, calibration, _document = _borrowed_source_export_fixture( + tmp_path, embedded_tiff=True + ) + rows = _external_rows(builder.build(batch, _settings(), channels).image_table) + assert tuple(row["Image_Frame_IllumDNA"] for row in rows) == (0, 1) + assert ( + tuple(row["Image_FileName_IllumDNA"] for row in rows) == (calibration.name,) * 2 + ) + assert ( + tuple( + (row["Image_Height_IllumDNA"], row["Image_Width_IllumDNA"]) for row in rows + ) + == ((5, 7),) * 2 + ) + + +def test_source_bound_export_does_not_cross_declared_execution_scope( + tmp_path: Path, +) -> None: + from openhcs.core.component_group_scope import RuntimeExecutionAxisScope + + builder, batch, channels, _calibration, _document = _borrowed_source_export_fixture( + tmp_path, calibration_scope=RuntimeExecutionAxisScope("B01") + ) + rows = _external_rows(builder.build(batch, _settings(), channels).image_table) + assert len(rows) == 2 + assert all(row.get("Image_FileName_IllumDNA") is None for row in rows) + assert all(row["Image_FileName_DNA"] for row in rows) + + def test_measurement_provenance_projects_exact_named_contributors_by_site( tmp_path: Path, ) -> None: diff --git a/tests/unit/test_image_file_serialization.py b/tests/unit/test_image_file_serialization.py index b9dcee0ffe..37aa545030 100644 --- a/tests/unit/test_image_file_serialization.py +++ b/tests/unit/test_image_file_serialization.py @@ -7,6 +7,7 @@ from openhcs.core.image_file_serialization import ( ImageFileFormat, + ImageFileSourceMetadata, NumpyImageFileFormat, PngImageFileFormat, TiffImageFileFormat, @@ -124,6 +125,7 @@ def test_tiff_source_metadata_reads_dtype_and_declared_scale_without_imageio( assert metadata.source_dtype == np.dtype(np.uint16) assert metadata.intensity_scale == 4095.0 + assert metadata.image_shape_yx == (1, 2) assert metadata.pixel_semantics.channel_axis is None assert metadata.pixel_semantics.channel_count is None @@ -150,6 +152,7 @@ def test_tiff_source_metadata_reads_rgb_semantics_without_generic_reopen( assert metadata.intensity_scale == 255.0 assert metadata.pixel_semantics.channel_axis == -1 assert metadata.pixel_semantics.channel_count == 3 + assert metadata.image_shape_yx == (4, 5) assert metadata.pixel_semantics.validated_channel_axis(tifffile.imread(path)) == -1 @@ -177,6 +180,71 @@ def test_required_source_metadata_does_not_reuse_replaced_header( assert require_image_file_source_metadata(path).pixel_semantics.channel_axis is None assert image_file_source_metadata(path).pixel_semantics.channel_axis is None + assert require_image_file_source_metadata(path).image_shape_yx == (4, 5) + + +def test_numpy_source_header_owns_shape_dtype_and_revision_without_pixel_decode( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "illum.npy" + np.save(path, np.zeros((5, 7), dtype=np.float32)) + original_load = np.load + calls = [] + + def load_header(file, *, mmap_mode, allow_pickle): + assert mmap_mode == "r" + assert allow_pickle is False + calls.append(file) + return original_load(file, mmap_mode=mmap_mode, allow_pickle=allow_pickle) + + monkeypatch.setattr(np, "load", load_header) + metadata = require_image_file_source_metadata(path) + assert metadata.image_shape_yx == (5, 7) + assert metadata.source_dtype == np.dtype(np.float32) + assert require_image_file_source_metadata(path) is metadata + assert len(calls) == 1 + np.save(path, np.zeros((3, 4), dtype=np.uint16)) + replaced = require_image_file_source_metadata(path) + assert replaced.image_shape_yx == (3, 4) + assert replaced.source_dtype == np.dtype(np.uint16) + assert len(calls) == 2 + + +@pytest.mark.parametrize( + "shape", ((1,), (1, 2, 3), (0, 2), (-1, 2), (True, 2), (1.0, 2)) +) +def test_source_header_refuses_invalid_declared_yx_dimensions(shape) -> None: + with pytest.raises(ValueError, match="two positive integer dimensions"): + ImageFileSourceMetadata(image_shape_yx=shape) + + +def test_numpy_non_image_header_does_not_infer_spatial_axes(tmp_path) -> None: + for name, shape in (("scalar", ()), ("volume", (2, 5, 7)), ("color", (5, 7, 3))): + path = tmp_path / f"{name}.npy" + np.save(path, np.zeros(shape, dtype=np.float32)) + metadata = require_image_file_source_metadata(path) + assert metadata.image_shape_yx is None + assert metadata.source_frame_shape is None + with pytest.raises(ValueError, match="container-frame mapping"): + metadata.frame_for_source_indices((0,)) + + +def test_tiff_header_owns_selected_frame_grid_and_bounds(tmp_path) -> None: + path = tmp_path / "frames.tif" + tifffile.imwrite( + path, + np.zeros((2, 3, 5, 7), dtype=np.uint16), + photometric="minisblack", + metadata={"axes": "TZYX"}, + ) + metadata = require_image_file_source_metadata(path) + assert metadata.image_shape_yx == (5, 7) + assert metadata.source_frame_shape == (2, 3) + assert metadata.frame_for_source_indices((1, 2)) == 5 + with pytest.raises(ValueError, match="complete container-frame mapping"): + metadata.frame_for_source_indices((1,)) + with pytest.raises(ValueError, match="outside dimension"): + metadata.frame_for_source_indices((2, 0)) def test_tiff_source_metadata_uses_declared_planar_sample_axis(tmp_path) -> None: From c729c7fb9379321f3b7f71a1a4afdb9930377097 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 06:50:12 -0400 Subject: [PATCH 058/125] refactor: derive header image layout from pixel semantics owner Share declared channel-axis validation between loaded pixels and header geometry, preserving the no-axis early return and refusing invalid negative axes instead of wrapping them. Remove the foreign optional-field probe from the generic image format reader. Refs #444. --- openhcs/core/image_file_serialization.py | 27 +++++++++--- tests/unit/test_image_file_serialization.py | 48 +++++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/openhcs/core/image_file_serialization.py b/openhcs/core/image_file_serialization.py index 4ab05b2b94..0b2593fa72 100644 --- a/openhcs/core/image_file_serialization.py +++ b/openhcs/core/image_file_serialization.py @@ -47,6 +47,26 @@ def validated_channel_axis(self, payload: Any) -> int | None: if axis is None: return None shape = tuple(int(value) for value in np.shape(image_payload_data(payload))) + return self._validated_channel_axis_for_shape(axis, shape) + + def image_shape_yx_for_shape( + self, + shape: tuple[int, ...], + ) -> tuple[int, int] | None: + """Project format-declared channel bands without inferring image axes.""" + + axis = self.channel_axis + if axis is not None: + self._validated_channel_axis_for_shape(axis, shape) + normalized = axis if axis >= 0 else len(shape) + axis + shape = shape[:normalized] + shape[normalized + 1 :] + return shape if len(shape) == 2 else None + + def _validated_channel_axis_for_shape( + self, + axis: int, + shape: tuple[int, ...], + ) -> int: normalized = axis if axis >= 0 else len(shape) + axis if normalized < 0 or normalized >= len(shape): raise ValueError( @@ -279,9 +299,6 @@ def _read_source_metadata(cls, path: Path) -> ImageFileSourceMetadata: dtype = properties.dtype pixel_semantics = cls.require_pixel_semantics(path) shape = tuple(properties.shape) - if pixel_semantics.channel_axis is not None: - axis = pixel_semantics.channel_axis % len(shape) - shape = shape[:axis] + shape[axis + 1 :] return ImageFileSourceMetadata( source_dtype=dtype, intensity_scale=( @@ -289,8 +306,8 @@ def _read_source_metadata(cls, path: Path) -> ImageFileSourceMetadata: or image_intensity_scale_for_dtype(dtype) ), pixel_semantics=pixel_semantics, - image_shape_yx=( - tuple(int(value) for value in shape) if len(shape) == 2 else None + image_shape_yx=pixel_semantics.image_shape_yx_for_shape( + tuple(int(value) for value in shape) ), ) diff --git a/tests/unit/test_image_file_serialization.py b/tests/unit/test_image_file_serialization.py index 37aa545030..5646baefa8 100644 --- a/tests/unit/test_image_file_serialization.py +++ b/tests/unit/test_image_file_serialization.py @@ -10,6 +10,7 @@ ImageFileSourceMetadata, NumpyImageFileFormat, PngImageFileFormat, + SourceImagePixelSemantics, TiffImageFileFormat, image_file_source_metadata, prepare_disk_image_payloads, @@ -218,6 +219,53 @@ def test_source_header_refuses_invalid_declared_yx_dimensions(shape) -> None: ImageFileSourceMetadata(image_shape_yx=shape) +@pytest.mark.parametrize( + "axis,shape,expected", + ( + (-1, (5, 7, 3), (5, 7)), + (0, (3, 5, 7), (5, 7)), + (None, (5, 7), (5, 7)), + (None, (5, 7, 3), None), + ), +) +def test_format_declared_image_shape_shares_loaded_channel_validation( + axis, shape, expected +) -> None: + semantics = SourceImagePixelSemantics(axis, 3 if axis is not None else None) + assert semantics.image_shape_yx_for_shape(shape) == expected + assert semantics.validated_channel_axis(np.zeros(shape, dtype=np.uint8)) == axis + + +@pytest.mark.parametrize("shape", ((5, 7, 4), (5, 7), ())) +def test_source_header_and_loaded_pixels_refuse_same_invalid_channel_layout( + shape, +) -> None: + semantics = SourceImagePixelSemantics(2, 3) + with pytest.raises(ValueError): + semantics.image_shape_yx_for_shape(shape) + with pytest.raises(ValueError): + semantics.validated_channel_axis(np.zeros(shape, dtype=np.uint8)) + + +def test_no_channel_declaration_does_not_inspect_payload(monkeypatch) -> None: + from openhcs.core import image_file_serialization + + monkeypatch.setattr( + image_file_serialization, + "image_payload_data", + lambda _payload: pytest.fail("No-axis guard must precede payload inspection"), + ) + assert SourceImagePixelSemantics().validated_channel_axis(object()) is None + + +def test_declared_negative_channel_axis_is_not_wrapped_into_valid_range() -> None: + semantics = SourceImagePixelSemantics(-4, 3) + with pytest.raises(ValueError, match="axis -4 is invalid"): + semantics.image_shape_yx_for_shape((5, 7, 3)) + with pytest.raises(ValueError, match="axis -4 is invalid"): + semantics.validated_channel_axis(np.zeros((5, 7, 3))) + + def test_numpy_non_image_header_does_not_infer_spatial_axes(tmp_path) -> None: for name, shape in (("scalar", ()), ("volume", (2, 5, 7)), ("color", (5, 7, 3))): path = tmp_path / f"{name}.npy" From d4e800a7552623c8d5a55522aadea42728ecea6e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 07:06:42 -0400 Subject: [PATCH 059/125] fix: honor declared composition for unannotated raw ABI --- openhcs/core/callable_contract.py | 5 ++ tests/unit/test_callable_abi_preparation.py | 67 +++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/openhcs/core/callable_contract.py b/openhcs/core/callable_contract.py index ad74e85f5e..18e75a7f7f 100644 --- a/openhcs/core/callable_contract.py +++ b/openhcs/core/callable_contract.py @@ -445,6 +445,11 @@ def raw_main_flow_call_argument(self, source_payload: Any, signature: inspect.Si primary_name = self.primary_input_name(signature.parameters) annotation = None if primary_name is None else signature.parameters[primary_name].annotation + if ( + annotation is inspect.Parameter.empty + and self.image_payload_consumption is ImagePayloadConsumption.COMPOSED + ): + return source_payload return source_payload if any( isinstance(source_payload, argument_type) for argument_type in self._nominal_argument_types(annotation) diff --git a/tests/unit/test_callable_abi_preparation.py b/tests/unit/test_callable_abi_preparation.py index 8b050ad204..dd3f1aa8e9 100644 --- a/tests/unit/test_callable_abi_preparation.py +++ b/tests/unit/test_callable_abi_preparation.py @@ -6,6 +6,7 @@ from dataclasses import replace from functools import wraps from unittest.mock import Mock +from typing import Any import cloudpickle import numpy as np @@ -22,6 +23,8 @@ from openhcs.core.function_reference import ImportableFunctionReference from openhcs.core.processing_preparation import CallablePreparation, PreparationCacheBatch from openhcs.core.runtime_image_values import ImagePayloadMetadata, image_payload_data +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.pipeline.function_contracts import composed_image_payload from openhcs.processing.backends.lib_registry.registry_service import RegistryService from openhcs.core.runtime_batch_contracts import ( Pure2DSliceBatchExecutor, RuntimeBatchExecutionDomain, @@ -37,6 +40,70 @@ def raw_numeric(image: np.ndarray, scale: float = 2.0) -> np.ndarray: return image * scale +@composed_image_payload +def unannotated_composed_echo(image): + return image + + +@pytest.mark.parametrize( + ("composed", "annotation", "retains_payload"), + ((True, inspect.Parameter.empty, True), + (False, inspect.Parameter.empty, False), + (True, np.ndarray, False), + (True, Any, False), + (True, ArrayPayload, True)), +) +def test_composition_default_respects_explicit_raw_carrier_boundary( + composed, annotation, retains_payload, fake_preparation, +): + received = [] + def raw(image): + received.append(image) + return image + if annotation is not inspect.Parameter.empty: + raw.__annotations__["image"] = annotation + if composed: + composed_image_payload(raw) + contract = CallableContract.from_prepared_callable(raw) + pixels = np.arange(20, dtype=np.float32).reshape(1, 4, 5) + mask = np.ones((1, 4, 5), dtype=bool) + mask[0, 0, 0] = False + source = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_image_names=("FITC",), + ).payload_with(pixels, mask) + expected = source if retains_payload else pixels + + result = RuntimeCallablePolicy().contract_invocation(contract, raw, source, {}).call() + + assert result is expected + assert received[0] is expected + assert source.metadata.plane_axis is RuntimePlaneAxis.SOURCE_BINDING + np.testing.assert_array_equal(source.mask, mask) + + +@pytest.mark.parametrize("serializer", (pickle, cloudpickle)) +def test_prepared_composition_default_survives_transport_without_introspection( + serializer, fake_preparation, monkeypatch, +): + contract = CallableContract.from_prepared_callable(unannotated_composed_echo) + reference = ImportableFunctionReference( + import_identity=CallableImportIdentity.from_callable(unannotated_composed_echo), + composite_key="test:unannotated_composed_echo", metadata=contract.metadata, + ) + restored = CallableContract.from_callable(serializer.loads(serializer.dumps(reference))) + def forbidden(*args, **kwargs): + pytest.fail("Prepared composed ABI must not inspect live callable declarations") + monkeypatch.setattr(contract_module, "get_type_hints", forbidden) + monkeypatch.setattr(inspect, "signature", forbidden) + payload = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_image_names=("FITC",), + ).payload_with(np.ones((1, 4, 5)), np.ones((1, 4, 5), dtype=bool)) + for _ in range(40): + assert restored.raw_main_flow_call_argument(payload) is payload + + @pytest.fixture def fake_preparation(monkeypatch): prepare = Mock() From 424bd2029bc21af0626d74356d06cfe075d25363 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 07:41:29 -0400 Subject: [PATCH 060/125] docs: record thread-owned attribution and actual geometry ABI loss (#384, #450) --- .../shared_runtime_plumbing_384_20261001.rst | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 29acca111b..5941ad53c8 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -945,3 +945,76 @@ recording/export ceiling of 2.4189 seconds; a two-second saving requires about Receipts are ``openhcs-measurement-export-declared-state-profile-20261002`` and ``openhcs-measurement-export-declared-fields-alias-control-20261002.json`` under ``/var/tmp``. The original four-snapshot and raw-heap failures remain retained. + + +Current-main integration and recording attribution (2026-10-02) +--------------------------------------------------------------- + +Main was normally merged through 3cb701770 at 6097ec4ff. Independent PR447 +merged the source-only CPA metadata repair and closed issue444. Its own +220 controls, original architecture check, public Advanced run and independent +scientific/source-identity comparison are documented in +``cpa-secondary-source-metadata-444.rst``. The larger branch still has open +acceptance gates and remains draft. + +The main58ee singleton controls exposed a real prepared-ABI integration bug: +unannotated COMPOSED functions lost their nominal carrier. d4e800a75 derives +that default from the existing consumption declaration while preserving +explicit ndarray/Any/nominal ABI precedence. 136 isolated and249 integrated +controls pass; the original scoped architecture check has zero increases. + +A fresh frozen746 Advanced run completed with compilation1.7983s, +execution9.4002s and total12.3852s, default observers and startup excluded. +Its scientific database comparison has zero differences. All90 previously +NULL calibration fields are present;70 match literally and20 native staging +PathName/URL strings are proven to identify the same source files using +same-inode, resolved-source and original native input-inventory SHA checks. +The raw path-string comparison remains separatelyRED. This is one observation, +not a performance benefit: colocalization and one primary identification step +were slower than frozen919, while database export changed1.8378 to1.7538s. + +The first nine-recorder cProfile diagnostic is scopeRED. It omitted the +existing monitoring callback thread filter and attributed background sleeps +to arbitrary numeric callers. A separate companion-thread experiment +reproduced the error and confirmed the existing worker-profiling policy +removes it after every profiler enable. No production workaround or duplicate +issue for the already-fixed profiler infrastructure was introduced. + +The corrected frozen919 diagnostic has zero foreign sleep/server/poller +events, exact original IFC CSV bytes, and zero scientific differences against +native CP for1800rows. Its disjoint recording attribution is diagnostic only: + +=============================== ============ +Region Profile time +=============================== ============ +Centroid calculation 0.6766s +Sparse relationship row assembly 0.4404s +Repeated axis-domain scans 0.2159s +Other table assembly 0.3644s +Ownership validation 0.000027s +Measurement storage 0.0123s +=============================== ============ + +Centroid-to-IJV conversion is nested within centroid calculation and must not +be added to it. Storage/ownership are immaterial targets for these recording +calls; centroid-only cannot deliver the multi-second goal. Recording plus +nonserialization export remains an optimistic2.4189s ceiling, not a measured +counterfactual improvement. + +A real source-born3D diagnostic on isolated8dec failed mask alignment at +step18. It observed300 source births and18 completed producer groups with +no observation errors. RescaledDNA starts with60 correlated physical Z rows; +Resize preserves60planes and changes onlyXY but drops the source domain. +Resize after ImageMath repeats the loss. Source audit identifies bare ndarray +annotations on the public Resize entries despite their shared implementation +requiring metadata/masks. Direct unwrapped tests bypassed this boundary. +Issue450 is formally linked to PR394; truthful RuntimeArrayData declarations, +prepared-contract tests and a fresh production rerun are required. No source +facts are injected into legacy captures, and no3D speedup/parity claim is made. + +Retained thread-scope/science receipts: +``/var/tmp/openhcs-nine-recorders-profile-thread-scope-control-20261002.json``, +``/var/tmp/openhcs-nine-recorders-profile-thread-owned-science-20261002.json`` +and ``/var/tmp/openhcs-nine-recorders-thread-owned-disjoint-analysis-20261002.json``. +Actual failed3D proof: +``/var/tmp/openhcs-spatial-plane-sourceborn-diagnostic-v2-20261002/observations.json``. From bae7f072bf88f36f2811c004560d8d6c23ba37ae Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 08:03:41 -0400 Subject: [PATCH 061/125] docs: record current 3D evidence and reject unused spatial prototype (#450) --- .../shared_runtime_plumbing_384_20261001.rst | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 5941ad53c8..8d2ab12902 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -984,16 +984,16 @@ The corrected frozen919 diagnostic has zero foreign sleep/server/poller events, exact original IFC CSV bytes, and zero scientific differences against native CP for1800rows. Its disjoint recording attribution is diagnostic only: -=============================== ============ -Region Profile time -=============================== ============ -Centroid calculation 0.6766s -Sparse relationship row assembly 0.4404s -Repeated axis-domain scans 0.2159s -Other table assembly 0.3644s -Ownership validation 0.000027s -Measurement storage 0.0123s -=============================== ============ +================================ ============ +Region Profile time +================================ ============ +Centroid calculation 0.6766s +Sparse relationship row assembly 0.4404s +Repeated axis-domain scans 0.2159s +Other table assembly 0.3644s +Ownership validation 0.000027s +Measurement storage 0.0123s +================================ ============ Centroid-to-IJV conversion is nested within centroid calculation and must not be added to it. Storage/ownership are immaterial targets for these recording @@ -1018,3 +1018,51 @@ Retained thread-scope/science receipts: and ``/var/tmp/openhcs-nine-recorders-thread-owned-disjoint-analysis-20261002.json``. Actual failed3D proof: ``/var/tmp/openhcs-spatial-plane-sourceborn-diagnostic-v2-20261002/observations.json``. + + +Actual 3D production completion and remaining qualification (2026-10-02) +---------------------------------------------------------------------- + +The isolated physical-plane prototype at aba4bfaf3 normally incorporates this +branch through424bd2029. Its truthful Resize ABI declarations and prepared +FULL_STACK controls pass438 tests. The actual new production run completes +with no observer errors,780 source-plane births and32 producer groups. Resize +now retains the runtime stack axis and all60 correlated physical rows; +MonolayerMask and the primary input consequently use the existing declared +stack-binding path. The diagnostic controller had required the different +opaque-image fallback's hook. Its original observation gate remains RED; +zero calls to that fallback is not a production failure or evidence of lost +metadata. The actual route is separately adjudicated from retained producer +observations and the existing binding declaration. + +Independent unchanged saved-output comparison passes all6 CSVs and120 TIFFs +exactly against the specified retained ordinary reference: headers/cells and +image dtype/shape/pixels. Receipt: +``/var/tmp/openhcs-spatial-plane-aba4-independent-historical-science-20261002.json``. +Diagnostic clocks are compilation2.1216s, execution9.7020s and total12.7136s; +they exclude server startup but do not establish a new performance benefit +or a current native CP ratio. Original R1 passes within its unchanged budget; +original R0 retains the quoted-Iterable finding. Independent actual RGB-volume +controls show identical pixels/masks but incorrect XY metadata in the +prototype: (3,3) instead of the declared channel-aware (2,3). Baseline passes. +The separate 2D RGB mask failure also occurs on baseline and is not claimed +as a new regression. + +The decisive route reassessment rejects promotion of the broader physical-Z +prototype. Preserving Resize's runtime stack carrier makes the existing typed +binding sufficient here; the new opaque physical-plane binding has no actual +production consumer in this run. Synthetic controls and successful saved +science do not establish its runtime payoff. Further prototype implementation +is stopped; the next counterfactual applies only truthful carrier declarations +and existing prepared metadata/mask/stack controls to current PR394. Actual +production execution must identify any remaining necessary repair. The +original failed observer gate and the RGB counterexample remain retained; +no source facts are injected and no gate is retroactively called PASS. + +PR394 normally incorporates latest main4754fbe2b at2aa930eb2. The added Napari +integration controls pass183 cases and fail the existing offscreen window +width assertion (1260 versus0). The exact isolated assertion also fails on +clean main4754fbe2b with the same values; it remains a separately recorded +inherited gate failure, not a passing full suite. Logs: +``/var/tmp/openhcs-pr394-latest-main4754-napari-controls-20261002.log`` and +``/var/tmp/openhcs-main4754-napari-layout-isolated-20261002.log``. From fecf3114aeec77931718e7da4ea885c9aa7cdfe4 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 08:08:30 -0400 Subject: [PATCH 062/125] Preserve typed geometry carriers at prepared call boundaries (#450) Declare the actual RuntimeArrayData inputs and returns of Resize and ResizeVolumetric, and the actual MaskImage return family. Preserve existing masks, source provenance and runtime/opaque domains through the prepared FULL_STACK executor without changing numerical or contextualization algorithms. --- .../backends/cellprofiler/image_geometry.py | 10 +- ...lprofiler_geometry_prepared_payload_abi.py | 127 ++++++++++++++++++ 2 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py diff --git a/openhcs/processing/backends/cellprofiler/image_geometry.py b/openhcs/processing/backends/cellprofiler/image_geometry.py index 37b24b3ce3..f51560c8a8 100644 --- a/openhcs/processing/backends/cellprofiler/image_geometry.py +++ b/openhcs/processing/backends/cellprofiler/image_geometry.py @@ -1098,7 +1098,7 @@ def mask_image( mask_source: MaskSource = MaskSource.IMAGE, invert_mask: bool = False, binary_threshold: float = 0.5, -) -> np.ndarray: +) -> RuntimeArrayData: """Mask an image using CellProfiler image/object mask semantics. Args: @@ -1276,14 +1276,14 @@ def tile( @numpy(contract=ProcessingContract.PURE_2D) def resize( - image: np.ndarray, + image: RuntimeArrayData, resize_method: ResizeMethod = ResizeMethod.BY_FACTOR, resizing_factor_x: float = 0.25, resizing_factor_y: float = 0.25, specific_width: int = 100, specific_height: int = 100, interpolation: InterpolationMethod = InterpolationMethod.NEAREST_NEIGHBOR, -) -> np.ndarray: +) -> RuntimeArrayData: """Resize a CellProfiler image plane by factor or explicit dimensions.""" pixels = image_payload_data(image) geometry = ResizeGeometry.from_parameters( @@ -1298,7 +1298,7 @@ def resize( @numpy(contract=ProcessingContract.PURE_3D) def resize_volumetric( - image: np.ndarray, + image: RuntimeArrayData, resize_method: ResizeMethod = ResizeMethod.BY_FACTOR, resizing_factor_x: float = 0.25, resizing_factor_y: float = 0.25, @@ -1307,7 +1307,7 @@ def resize_volumetric( specific_height: int = 100, specific_planes: int = 10, interpolation: InterpolationMethod = InterpolationMethod.NEAREST_NEIGHBOR, -) -> np.ndarray: +) -> RuntimeArrayData: """Resize a CellProfiler ZYX image volume by factor or explicit dimensions.""" pixels = image_payload_data(image) geometry = ResizeGeometry.from_trailing_spatial_parameters( diff --git a/tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py b/tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py new file mode 100644 index 0000000000..933738e65e --- /dev/null +++ b/tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py @@ -0,0 +1,127 @@ +"""Geometry declarations preserve the payload their raw implementations consume.""" + +from dataclasses import replace +from typing import get_type_hints + +import numpy as np +import pytest + +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.callable_contract import CallableContract +from openhcs.core.runtime_array_values import RuntimeArrayData +from openhcs.core.runtime_image_values import ( + ImagePayloadMetadata, + image_payload_data, + image_payload_mask, + image_payload_metadata, +) +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.interop.cellprofiler.runtime.adapter import CellProfilerRuntimeAdapter +from openhcs.interop.cellprofiler.runtime.function_contract_execution import ( + CellProfilerFunctionContractExecutor, +) +from openhcs.processing.backends.cellprofiler.image_geometry import ( + mask_image, + resize, + resize_volumetric, +) + + +def _prepared_contract(func): + contract = CallableContract.from_callable(func).with_prepared_signature() + return replace( + contract, + metadata=replace( + contract.metadata, + runtime_adapter=CellProfilerRuntimeAdapter.runtime_adapter_spec(), + ), + ) + + +@pytest.mark.parametrize("func", (resize, resize_volumetric)) +def test_prepared_resize_preserves_typed_primary_and_bare_array(func): + pixels = np.arange(24, dtype=np.float32).reshape(4, 6) + source = ImagePayloadMetadata( + source_path="/images/input.tif", source_image_names=("Original",), + ).payload_with(pixels, pixels % 2 == 0) + contract = _prepared_contract(func) + + assert contract.raw_main_flow_call_argument(source) is source + assert contract.raw_main_flow_call_argument(pixels) is pixels + + +@pytest.mark.parametrize("axis", (None, RuntimePlaneAxis.RUNTIME_SLICE)) +@pytest.mark.parametrize("z_factor", (1.0, 2.0)) +def test_prepared_full_stack_resize_preserves_existing_metadata_and_mask(axis, z_factor): + planes = tuple( + ImagePayloadMetadata( + source_path=f"/images/z{index}.tif", + source_image_names=("Original",), + source_component_metadata={"z_index": index, "timepoint": 1, "well": "W001"}, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(2, 3), source_shape_yx=(8, 10), + ), + ).payload_with(np.full((4, 6), index, dtype=np.float32), None) + for index in (3, 1, 2) + ) + metadata = ImagePayloadMetadata.compose(planes).replace_fields(plane_axis=axis) + pixels = np.stack(tuple(image_payload_data(plane) for plane in planes)) + mask = np.ones(pixels.shape, dtype=bool) + mask[:, 0, 0] = False + source = metadata.payload_with(pixels, mask) + contract = _prepared_contract(resize_volumetric) + canonical = contract.resolve_canonical_raw_callable() + kwargs = dict( + resizing_factor_x=0.5, resizing_factor_y=0.5, resizing_factor_z=z_factor, + ) + expected = canonical(source, **kwargs) + + result = CellProfilerFunctionContractExecutor().execute( + contract, canonical, source, kwargs, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(expected)) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(expected)) + assert image_payload_metadata(result) == image_payload_metadata(expected) + assert image_payload_metadata(result).plane_axis is axis + assert image_payload_metadata(result).source_image_paths == metadata.source_image_paths + assert image_payload_metadata(result).source_provenance == metadata.source_provenance + assert image_payload_metadata(result).source_spatial_domain.source_shape_yx == (2, 3) + assert image_payload_metadata(source) is metadata + np.testing.assert_array_equal(image_payload_data(source), pixels) + np.testing.assert_array_equal(image_payload_mask(source), mask) + + +@pytest.mark.parametrize("func", (resize, resize_volumetric)) +def test_prepared_resize_bare_pixels_preserve_canonical_return_contract(func): + pixels = np.arange(24, dtype=np.float32).reshape(4, 6) + contract = _prepared_contract(func) + canonical = contract.resolve_canonical_raw_callable() + kwargs = dict(resizing_factor_x=0.5, resizing_factor_y=0.5) + if func is resize_volumetric: + kwargs["resizing_factor_z"] = 1.0 + expected = canonical(pixels, **kwargs) + + result = CellProfilerFunctionContractExecutor().execute( + contract, canonical, pixels, kwargs, + execution_mode=ImagePayloadExecutionMode.FULL_STACK, + ) + + assert isinstance(result, RuntimeArrayData) + assert type(result) is type(expected) + np.testing.assert_array_equal(image_payload_data(result), image_payload_data(expected)) + np.testing.assert_array_equal(image_payload_mask(result), image_payload_mask(expected)) + assert image_payload_metadata(result) == image_payload_metadata(expected) + + +def test_mask_image_declares_its_actual_nominal_return_family(): + contract = _prepared_contract(mask_image) + canonical = contract.resolve_canonical_raw_callable() + assert get_type_hints(canonical)["return"] == RuntimeArrayData + pixels = np.arange(12, dtype=np.float32).reshape(3, 4) + source = ImagePayloadMetadata(source_path="/images/input.tif").payload_with(pixels) + result = canonical(source, pixels > 3) + assert isinstance(result, RuntimeArrayData) + assert image_payload_metadata(result).source_image_paths == ("/images/input.tif",) From 3cd113f1d57064a7911a6bf747425c45fcb9a1ba Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 08:19:16 -0400 Subject: [PATCH 063/125] docs: qualify narrow geometry fix and fresh native 3D comparison (#450) --- .../shared_runtime_plumbing_384_20261001.rst | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 8d2ab12902..6ab20bfa48 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1066,3 +1066,60 @@ clean main4754fbe2b with the same values; it remains a separately recorded inherited gate failure, not a passing full suite. Logs: ``/var/tmp/openhcs-pr394-latest-main4754-napari-controls-20261002.log`` and ``/var/tmp/openhcs-main4754-napari-layout-isolated-20261002.log``. + + +Qualified narrow carrier repair and fresh native 3D (2026-10-02) +--------------------------------------------------------------- + +The rejected physical-plane prototype is absent from PR394. Narrow commit +b7640ad61 on isolated main-synchronized2aa930eb2 changes only five geometry +annotations; no numerical body, domain, HoleRemoval or renderer changes. +Nine prepared/FULL_STACK controls establish7 failures before and9 passes +after, preserving typed masks, runtime/opaque domains and source provenance. +The related suite passes148 tests; unchanged original R0/R1 both pass within +their original budgets. Integration fecf3114a has exactly the same production +and test Git trees as the qualified isolated source; only validation prose +differs. The fix is formally carried by PR394 for issue450. + +The actual uninstrumented default3D pipeline completes, with exact6CSV and +120TIFF saved science against the same retained reference. All180 actual +source-plane references belong to exactly3 physical TIFFs with indices0..59 +and unchanged source-file SHA values. Both fresh native CP4.2.8.1 measured +repeats pass the existing strict6table/150key/2825fact comparison, exact two +logical uint16(60,256,256) label volumes and complete physical-file coverage. +Native comparison to the prototype plus exact prototype/reference/narrow +comparisons establishes transitive narrow parity; no direct narrow/native +comparison is claimed. The original comparison-controller API failure +(tuple subtraction from frozenset) remains RED; a separate recipe supplies +the required frozenset input views without changing any scientific gate. + +Fresh current main4754fbe2b also matches the same saved reference exactly. +All clocks below use CPU5 and one thread, with startup/warmup excluded. +OpenHCS includes default OUTCOMES and memory observation; native invocation +includes prepare_run, module execution, post_run and Measurements.close, +and excludes imports/JVM/pipeline loading plus its separate warmup. + +===================== ============ ============ ============ +Observation Compilation Execution Total +===================== ============ ============ ============ +Main4754 ordinary 1.6754s 10.0712s 12.4766s +Narrowb764 ordinary 1.6190s 8.3008s 10.7417s +Native measured0 excluded 14.3544s invocation +Native measured1 excluded 14.4375s invocation +===================== ============ ============ ============ + +The observed main/candidate difference is1.7704s execution and1.7349s total +for the full PR394 plus annotation repair. It is not evidence that the five +annotations alone save that time. One ordinary observation per source does +not establish repeatability or statistical significance. Native's two-run +mean is14.39595s, giving a descriptive1.7343x candidate execution ratio and +1.3402x whole-pipeline ratio under the explicitly different clock scopes. +The target gap remains: reaching2x requires about1.10s further execution +reduction, or3.54s whole-pipeline reduction. No goal completion is claimed. + +Receipts: +``/var/tmp/openhcs-geometry-narrow-original-guards-20261002.json``, +``/var/tmp/openhcs-geometry-narrow-b764-3d-ordinary-20261002/observations.json``, +``/var/tmp/openhcs-geometry-narrow-b764-independent-science-and-inputs-20261002.json``, +``/var/tmp/openhcs-main4754-independent-historical-3d-science-20261002.json`` and +``/var/tmp/openhcs-native-3d-aba4-qualification-20261002/fresh-native-scientific-comparison-supplemental.json``. From c5e15c84a33e77144f187e94f97b38e40755c33f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 08:58:34 -0400 Subject: [PATCH 064/125] docs: record four-run 3D timings and dominant plumbing frontier (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 6ab20bfa48..643f91f193 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1123,3 +1123,81 @@ Receipts: ``/var/tmp/openhcs-geometry-narrow-b764-independent-science-and-inputs-20261002.json``, ``/var/tmp/openhcs-main4754-independent-historical-3d-science-20261002.json`` and ``/var/tmp/openhcs-native-3d-aba4-qualification-20261002/fresh-native-scientific-comparison-supplemental.json``. + + +Four-run 3D comparison and dominant runtime frontier (2026-10-02) +---------------------------------------------------------------- + +The subsequent A1/B1/B2/A2 ordinary comparison supersedes the single-pair +ratios above. A is frozen clean main4754fbe2b; B is frozen narrowb7640ad61, +including the full PR394 runtime change and five truthful annotations. Each +run uses CPU5, one worker/thread, the unchanged public driver, default OUTCOMES +and memory observation. Mandatory server/library/kernel startup and shutdown +remain outside the pipeline clocks. All four observations preserve exact six +CSV tables and120 TIFFs against the same retained reference, unchanged source +and dependency freezes, and all180 source-plane references to the same three +physical volumes used by fresh native CP. + +======================= ============ ============ ============ +Observation Compilation Execution Total +======================= ============ ============ ============ +Main A1 1.6754s 10.0712s 12.4766s +Candidate B1 1.6190s 8.3008s 10.7417s +Candidate B2 1.7936s 8.7713s 11.3281s +Main A2 1.6213s 10.0910s 12.4374s +Main mean 1.6484s 10.0811s 12.4570s +Candidate mean 1.7063s 8.5360s 11.0349s +======================= ============ ============ ============ + +The descriptive mean reductions are1.5451s execution and1.4221s total. +Native warm invocation mean14.39595s gives1.6865x execution and1.3046x +total ratios under the previously stated clock scopes. Two observations per +source do not establish statistical significance or annotation-only causality. +The remaining2x gap is1.3381s execution and3.8369s total, larger than the +first-pair estimate. The complete independent receipt is +``/var/tmp/openhcs-3d-b764-abba-science-clocks-20261002.json``. + +A separate source-pinned diagnostic partitions8.3448s of execution into +4.0497s inside the callable boundary and4.2951s outside it. Callable time +includes decorated metadata/control work, not only numerical kernels. The +exclusive outer terms include stack loading0.5991s, unstacking0.2126s, +saving0.3487s, output identity0.2070s across1680 calls, image recording0.2333s, +CP image requests0.2012s, SaveImages invocation preparation0.3463s, metadata +finalization0.1112s, publication0.4795s and reconciliation0.2812s. These are +diagnostic spans, not accepted speedup measurements; required copying, +validation and I/O remain within them. + +The SaveImages preparation span stores memory artifacts and contextualizes +their values; it is not TIFF encoding. Actual artifact materialization has a +separate0.0831s envelope. Saved metadata contains120 SourceArtifactProjection +entries from the two SaveImages outputs. The generic main-flow metadata/VFS +reload branch is not the observed publication route. The actual joint route +is artifact invocation/store, successful materialization metadata, source +artifact projection, serialization, locked publication and reconciliation. + +Publication/finalization/identity alone has an optimistic1.0789s ceiling and +is rejected as insufficient to close the current execution gap. The broader +cohort/recording/invocation/publication envelope is3.0201s and requires at +least44.3% collapse before accounting for mandatory work. Existing whole-stack +cache lookup/storage is already below1ms; dictionary tuning or assuming every +input is restacked cannot support the target. The next admission requires an +actual saved joint leaf replay preserving metadata mutation, source correlation, +custom normalization, error ordering and pixel/mask isolation. No new mutable +identity cache or production optimization has yet been admitted by this ceiling. + +Diagnostic science and freeze validation: +``/var/tmp/openhcs-narrow450-phase-science-and-freeze-20261002.json``. +Global authority/consumer inventory and corrected route: +``/var/tmp/openhcs-b764-publication-authority-consumer-inventory-20261002.json`` +and ``/var/tmp/openhcs-b764-coupled-publication-source-frontier-v2-20261002.json``. + +Latest mainb68029c0c is normally merged atc0d78b62e. The initial MCP/agent +integration run passes131 cases and fails both MRO variants of a new cold +feedback fixture. Clean mainb680 reproduces both failures: its reporting thread +can observe published progress before the fake compiler sets its subsequent +``emitted`` marker. This identifies a fixture synchronization race, not a +demonstrated production race. The original candidate and clean-main failures +remain retained. Baseline causal receipt: +``/var/tmp/openhcs-main-b680-cold-feedback-race-baseline-20261002.json``. +Benchmark timings above remain pinned to their actual main4754/narrowb764 +sources; the newer MCP merge is not silently included in their evidence. From 3317bf9c2631324fefd1c92e63f74a13b0540f29 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 08:59:47 -0400 Subject: [PATCH 065/125] Synchronize cold progress fixture before publishing (#436) The SDK progress relay can run before emit_progress returns. Mark entry into held compilation before publishing, preserving the real progress release and both cooperative MRO assertions. --- tests/unit/agent/test_cold_inspection_feedback.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/agent/test_cold_inspection_feedback.py b/tests/unit/agent/test_cold_inspection_feedback.py index 57db9b0f62..32d6a7d586 100644 --- a/tests/unit/agent/test_cold_inspection_feedback.py +++ b/tests/unit/agent/test_cold_inspection_feedback.py @@ -63,7 +63,7 @@ def test_new_gateway_hook_cooperates_in_both_mros_before_terminal_result(tmp_pat """Independent audit capability needs no edits to service or MCP consumers.""" identity = ContextVar("cold-feedback-request", default="outside") calls, messages = [], [] - emitted = threading.Event() + compile_entered = threading.Event() release = threading.Event() error = ValueError("original declared compiler error") @@ -77,8 +77,8 @@ def compile(self, request): class DeclaredGateway(_FakeCompileInspectionGateway): def _compile(self, request): assert threading.current_thread() is threading.main_thread() + compile_entered.set() self.emit_progress(request) - emitted.set() assert release.wait(3), "Actual stage relay was not delivered during held work" if fails: raise error @@ -97,7 +97,7 @@ async def report_progress(self, progress, total=None, message=None): assert threading.current_thread() is not threading.main_thread() messages.append(message) if "compile: running: A01: compilation" in message: - assert emitted.is_set() + assert compile_entered.is_set() release.set() executor = McpTransportExecutor() @@ -105,7 +105,7 @@ async def report_progress(self, progress, total=None, message=None): try: for gateway_type in (Before, After): release.clear() - emitted.clear() + compile_entered.clear() gateway = gateway_type() service = ExecutionSessionService( path_policy=AgentPathPolicy.with_roots( From 7a835b48a0931e61bd6c0c6a00db6c1967b5b715 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 09:04:09 -0400 Subject: [PATCH 066/125] docs: qualify latest-main cold progress integration controls (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 643f91f193..3c6c7ed82e 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1201,3 +1201,26 @@ remain retained. Baseline causal receipt: ``/var/tmp/openhcs-main-b680-cold-feedback-race-baseline-20261002.json``. Benchmark timings above remain pinned to their actual main4754/narrowb764 sources; the newer MCP merge is not silently included in their evidence. + +Test-only integration3317bf9c2 sets the compiler-entry marker before publishing +progress. The actual reporting callback still releases held work; the original +three-second timeout, both cooperative MROs, ContextVar restoration, terminal +ordering and original compiler-error assertions are unchanged. No production +race workaround, retry or longer timeout is introduced. The clean-main repair +passes124 cold-feedback/agent-service controls; integrated PR394 then passes +all133 controls including the narrow prepared geometry ABI tests in13.72s. +Both initial failed runs remain retained. Integrated command, CPU3: + +.. code-block:: bash + + env PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/home/ts/code/projects/openhcs-shared-runtime-plumbing \ + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \ + NUMBA_NUM_THREADS=1 CUDA_VISIBLE_DEVICES='' taskset -c 3 \ + /home/ts/code/projects/openhcs/.venv/bin/python -m pytest -q \ + -p no:cacheprovider tests/unit/agent/test_cold_inspection_feedback.py \ + tests/unit/agent/test_agent_services.py \ + tests/unit/test_cellprofiler_geometry_prepared_payload_abi.py + +Pass log: +``/var/tmp/openhcs-pr394-main-b680-repaired-integration-controls-20261002.log``. From 789e3c3043685ef77bb7cc087dec202f3e565ce1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 09:53:42 -0400 Subject: [PATCH 067/125] test(mcp): scope resource fixtures and activate checkout imports (#457) --- .../test_mcp_process_resource_lifetime.py | 111 +++++++++--------- 1 file changed, 56 insertions(+), 55 deletions(-) diff --git a/tests/unit/agent/test_mcp_process_resource_lifetime.py b/tests/unit/agent/test_mcp_process_resource_lifetime.py index d64c219894..f497a95889 100644 --- a/tests/unit/agent/test_mcp_process_resource_lifetime.py +++ b/tests/unit/agent/test_mcp_process_resource_lifetime.py @@ -18,62 +18,63 @@ @pytest.mark.parametrize("failure", (None, ValueError("original resource failure"))) def test_original_resource_declaration_and_cooperative_close(monkeypatch, failure): - calls = [] - - class Gateway: - def dispose(self): - assert threading.current_thread() is threading.main_thread() - calls.append("gateway disposed") - - class Java: - def shutdown_jvm(self): + with monkeypatch.context() as resource_patch: + calls = [] + + class Gateway: + def dispose(self): + assert threading.current_thread() is threading.main_thread() + calls.append("gateway disposed") + + class Java: + def shutdown_jvm(self): + assert threading.current_thread() is threading.main_thread() + calls.append("JVM stopped") + + context = BioFormatsJavaContext(imagej_module=None, scyjava_module=Java()) + context.ij = Gateway() + resource_patch.setattr(BioFormatsJavaContext, "_instance", context) + resource_patch.setattr(backend_registry, "_cleanup_callbacks", []) + backend_registry.register_cleanup_callback(BioFormatsJavaContext.shutdown_instance) + + class AuditCloseCapability: + def close(self): + calls.append("cooperative enter") + super().close() + calls.append("cooperative exit") + + class AuditedExecutor(AuditCloseCapability, McpTransportExecutor): + pass + + executor = AuditedExecutor() + + def independent_resource(): + assert executor._closed + assert executor.active_futures() == () assert threading.current_thread() is threading.main_thread() - calls.append("JVM stopped") - - context = BioFormatsJavaContext(imagej_module=None, scyjava_module=Java()) - context.ij = Gateway() - monkeypatch.setattr(BioFormatsJavaContext, "_instance", context) - monkeypatch.setattr(backend_registry, "_cleanup_callbacks", []) - backend_registry.register_cleanup_callback(BioFormatsJavaContext.shutdown_instance) - - class AuditCloseCapability: - def close(self): - calls.append("cooperative enter") - super().close() - calls.append("cooperative exit") - - class AuditedExecutor(AuditCloseCapability, McpTransportExecutor): - pass - - executor = AuditedExecutor() - - def independent_resource(): - assert executor._closed - assert executor.active_futures() == () - assert threading.current_thread() is threading.main_thread() - calls.append("independent resource") - if failure is not None: - raise failure - - # A new resource needs only its original declaration, not an MCP edit. - backend_registry.register_cleanup_callback(independent_resource) - backend_registry.register_cleanup_callback(independent_resource) - if failure is None: - executor.close() - else: - with pytest.raises(ExceptionGroup) as caught: + calls.append("independent resource") + if failure is not None: + raise failure + + # A new resource needs only its original declaration, not an MCP edit. + backend_registry.register_cleanup_callback(independent_resource) + backend_registry.register_cleanup_callback(independent_resource) + if failure is None: executor.close() - assert caught.value.exceptions == (failure,) - assert BioFormatsJavaContext._instance is None - assert context.ij is None - assert calls[:4] == [ - "cooperative enter", "gateway disposed", "JVM stopped", "independent resource" - ] - assert calls.count("cooperative exit") == int(failure is None) - executor.close() - assert calls.count("gateway disposed") == 1 - assert calls.count("JVM stopped") == 1 - assert calls.count("independent resource") == 1 + else: + with pytest.raises(ExceptionGroup) as caught: + executor.close() + assert caught.value.exceptions == (failure,) + assert BioFormatsJavaContext._instance is None + assert context.ij is None + assert calls[:4] == [ + "cooperative enter", "gateway disposed", "JVM stopped", "independent resource" + ] + assert calls.count("cooperative exit") == int(failure is None) + executor.close() + assert calls.count("gateway disposed") == 1 + assert calls.count("JVM stopped") == 1 + assert calls.count("independent resource") == 1 def test_off_main_close_rejected_before_resource_release(monkeypatch): @@ -165,9 +166,9 @@ def test_real_sdk_stdio_declared_resource_closes_before_child_exit(tmp_path, fai script.write_text( f"import sys\nsys.path.insert(0, {str(source_root)!r})\n" "from pathlib import Path\nimport threading,inspect\n" + "from openhcs.mcp.stdio import McpStdioTransport\n" "from mcp.server.fastmcp import FastMCP\n" "from polystore import register_cleanup_callback\n" - "from openhcs.mcp.stdio import McpStdioTransport\n" f"assert Path(inspect.getfile(McpStdioTransport)).resolve().is_relative_to(Path({str(source_root)!r}))\n" "server = FastMCP('declared-resource-lifetime')\n" "@server.tool()\ndef health() -> dict[str, bool]:\n return {'ok': True}\n" From 5770763371820a74ce0b29dadc4fa16a7e98565a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 09:58:34 -0400 Subject: [PATCH 068/125] docs: retain rejected leaf routes and whole-worker evidence (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 3c6c7ed82e..9e9509716f 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1224,3 +1224,71 @@ Both initial failed runs remain retained. Integrated command, CPU3: Pass log: ``/var/tmp/openhcs-pr394-main-b680-repaired-integration-controls-20261002.log``. + +Saved-leaf rejection and whole-worker diagnostic, 2026-10-02 +---------------------------------------------------------- + +A second source-b764 diagnostic captures the actual contextualization, +normalization scopes, identity-cache preimages, serializer inputs and five +locked publication preimages. All39 admitted snapshots retain their original +metadata, source correlations and alias relations. Exact six CSV/120 TIFF +comparison against the same-head ordinary run and retained historical reference, +all180 physical input references and source/dependency/native freezes pass. +Injected capture work is separately timed; these are not accepted pipeline clocks. +Receipt: +``/var/tmp/openhcs-narrow450-joint-leaf-v3-science-and-freeze-20261002.json``. + +An exploratory metadata projection fusion passes the11 captured valid-input +leaf comparisons but saves only4.82ms across the five actual context calls; +normalization has no measured gain. Separately, malformed self spatial-domain +metadata combined with malformed source fallback spacing changes the first +exception. The original reports the spatial-domain failure; the candidate +reports invalid spacing. This is a real correctness failure, independent of a +historical test that observes normalization phase counts. The candidate is +rejected for both inadequate payoff and error-order drift, and is not included +in this branch. Original/candidate replay receipts: +``/var/tmp/openhcs-narrow450-joint-leaf-v3-original-replay-20261002.json`` and +``/var/tmp/openhcs-narrow450-joint-leaf-v3-fusion-replay-20261002.json``. +Malformed-input witness: +``/var/tmp/repro_narrow450_fusion_domain_spacing_order_20261002.py``. + +All five original locked publication transactions and both serializer calls +replay with exact ordered JSON results and argument after-state. Original replay +transaction medians sum to0.4646s; the observed callback sum is0.3760s and +lock/read/JSON-write residual0.0872s. Replacing three final transactions with one +would remove only about0.04s of this replay's I/O baseline. Standalone batching +is rejected as insufficient. These are saved-input replay clocks, not an +end-to-end speedup; original on-disk whitespace was not captured. Receipt: +``/var/tmp/openhcs-original-publication-v3-replay-20261002/receipt.json``. + +One further actual public-driver diagnostic uses the existing thread-owned +worker profiling policy, with no source edits or diagnostic monkeypatches. +The worker records26,443,742 calls, including5,280 slice-context projections, +25,304 source-metadata merges,55,173 source identity constructions and248 NumPy +stack calls. The next investigation follows repeated whole-image/plane +derivations across loading, invocation, saving and publication. It does not +assume every cache hit copies or every saved plane has independent pixels: +the observed SaveImages named values, VFS planes and cache share pixels, while +the observed Resize output has independent buffers. Profiling increases the +execution clock substantially; profile times must not be scaled into ordinary +costs or reported as benchmark speedups. Exact six CSV/120 TIFF and frozen source, +dependency, native and physical-input comparisons pass. Receipts: +``/var/tmp/openhcs-narrow450-existing-worker-profile-20261002/observations.json`` +and +``/var/tmp/openhcs-narrow450-existing-worker-profile-science-and-freeze-20261002.json``. + +Main17c630606 is normally merged ata6abc4418. Its new lifetime controls initially +produce140 passes, two SDK failures and one teardown error in the integrated +repository suite. The generated child imports PolyStore before activating the +selected OpenHCS checkout; the intentionally failing synthetic callback also +outlives its test into autouse cleanup. Test-only789e3c304 activates OpenHCS first +and restores the callback owner in a lexical monkeypatch context. Original +failure identity, cooperative close, main-thread checks and repeated-close +assertions remain unchanged. All142 lifetime/cold-feedback/agent-service/prepared +geometry controls then pass in25.24s. This is tracked separately by issue457; +closed issue455's installed production shutdown acceptance is not reopened. +Original RED and repaired logs: +``/var/tmp/openhcs-pr394-main-17c630606-integration-controls-20261002.log`` and +``/var/tmp/openhcs-pr394-main17-lifetime-fixture-repair-controls-20261002.log``. +AST/owner receipt: +``/var/tmp/openhcs-pr394-main17-lifetime-fixture-repair-20261002.json``. From 349c04d7484d1e89e872c3f757b428824ff636d0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 10:12:55 -0400 Subject: [PATCH 069/125] docs: link merged fixture repair and canonical owner census (#384) --- .../shared_runtime_plumbing_384_20261001.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 9e9509716f..c744eac2fc 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1292,3 +1292,21 @@ Original RED and repaired logs: ``/var/tmp/openhcs-pr394-main17-lifetime-fixture-repair-controls-20261002.log``. AST/owner receipt: ``/var/tmp/openhcs-pr394-main17-lifetime-fixture-repair-20261002.json``. + +The test-only repair ships independently in merged PR460, commitb495dbb13; +issue457 is formally linked and closed. Its exact clean-main branch passes21 +lifetime/stdio/dev-client controls through normal repository conftest collection +in13.21s. Main is normally merged back into this branch at01875a772; the merge +introduces no additional production diff. Clean-main qualification receipt: +``/var/tmp/openhcs-mcp-lifetime-fixture-457-main-qualification-20261002.json``. + +The next ownership census uses NRA's ModuleSyntaxIndex and canonical +CompactClassFamilyIndex on all703 OpenHCS production modules and the actual +63 PolyStore,12 python-introspect and6 metaclass-registry modules. All5506 +original class declarations remain represented;5493 join canonical family +declarations and13 remain unprojected OPEN. There are no parse errors. +Conditional/function-local binding, omitted numerical dependencies and native +runtime effects remain explicit proof limits. This is source discovery, not +an admitted architecture or equivalence proof. NRA revision0844525ec is pinned. +Receipt: +``/var/tmp/openhcs-b764-shared-derivation-nra-class-census-20261002.json``. From 51533665ead8ab415db19f13c41215cc3bda9c5c Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 10:50:56 -0400 Subject: [PATCH 070/125] docs: record reproduced public publication failure for #435 and current gates --- .../shared_runtime_plumbing_384_20261001.rst | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index c744eac2fc..9b2f801d5d 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1310,3 +1310,59 @@ runtime effects remain explicit proof limits. This is source discovery, not an admitted architecture or equivalence proof. NRA revision0844525ec is pinned. Receipt: ``/var/tmp/openhcs-b764-shared-derivation-nra-class-census-20261002.json``. + +Current-main integration and acceptance findings, 2026-10-02 +----------------------------------------------------------- + +Main cc9fcdfd4 is normally merged at 133da74ec. The affected native +presentation/MCP, agent-service, resource-lifetime and prepared-geometry suite +passes 150 cases in 17.75s. A separate current-source owner gate passes 73 +metadata-owner, live-mutation, provenance-constructor and resolution-snapshot +cases in 1.34s. These scoped gates do not erase the retained broader offscreen +window failure or qualify a new benchmark source. Logs: +``/var/tmp/openhcs-pr394-main-cc9fcdfd4-integration-controls-20261002.log`` and +``/var/tmp/openhcs-pr394-current-maincc9-source-owner-controls-20261002.log``. + +The original, unchanged whole-branch R0 is rerun against cc9fcdfd4 and +133da74ec with tool revision 3b03785f4. It exits 1. Its only positive measures +remain source_metadata's one TypeSwitch and three TypeSwitchArms; the removed +virtual_workspace_metadata classifier contributes minus one switch and minus +four arms. The ownership decision retains scalar admission on +SourceMetadataFields and durable rejection on DurableSourceMetadata: the +grammar belongs to the field owner, while canonical runtime spelling and +literal durable spelling remain substitutable policies. Reintroducing the +decoder's duplicate validator or moving the grammar to an unrelated utility +would contradict that ownership. This adjudicates the relocation's architecture; +it does not declare the per-file ratchet passed or alter its implementation, +budget, exclusions or observations. Primitive/subclass admission, reported-class +read ordering, scalar/container precedence, exact errors and mutable lifetime +remain separate behavior obligations covered by the owner controls. Raw report: +``/var/tmp/openhcs-pr394-current-maincc9-original-r0-20261002.log``. + +The metadata-only whole-query prototype is rejected immediately after saved +replay: its five captured context calls still take about 55ms, with no matched +speedup demonstrated. Valid saved leaf gates pass, but constructor/subclass, +malformed-input and live-mutation obligations remain unresolved. Neither its +callback-based query abstraction nor its manually replayed constructor effects +are promoted. A separate source trace falsifies wholesale stack-rebuild removal: +23 of 32 loads already hit the whole-stack cache and memory saves retain typed +payload references. The entire load/unstack/save/identity envelope is only +1.3674s before mandatory work, insufficient to close the 1.3381s execution gap. +The next route must span repeated source derivation across existing request, +recording, stack and publication owners. Receipts: +``/var/tmp/openhcs-whole-source-query-exploratory-REJECTED-20261002.json`` and +``/var/tmp/openhcs-b764-whole-cohort-cycle-dominant-route-review-20261002.json``. + +Issue 435's fresh public MCP synthetic acceptance is RED on frozen 133da74ec. +The original five processing steps are unchanged; only owned output paths and +the unused required DAPI binding differ. The first attempt stops before runtime +creation because native TCP lock paths do not follow XDG storage. A fresh, +separately retained attempt admits only its declared private data/control lock +paths, completes mandatory preparation and compilation, then fails the fifth +step at the unchanged duplicate source-projection guard. Its exact owned runtime +is closed through the public API and all SDK children are terminal. This is a +real workflow failure that the existing same-occurrence unit fixture did not +cover; installed acceptance remains open. No original biological replay or +performance claim follows from this synthetic check. Source/input/dependency, +native, request/reply and failure evidence: +``/var/tmp/openhcs-derived-role-435-synthetic-acceptance-v2-20261002``. From 0b6c665603cee6f9be89accf293b6adeb70cc189 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 11:22:05 -0400 Subject: [PATCH 071/125] fix: retain distinct declared image roles during automatic materialization Automatic source-identity image retention omitted the declared output qualifier, so two roles overwrote one unqualified file and failed strict publication. Share the existing filename identity projection across scalar and projected image outputs while preserving explicit export contracts. Refs #435. --- openhcs/core/artifacts.py | 24 ++ .../function_artifact_materialization.py | 3 + openhcs/processing/materialization/core.py | 66 ++++-- .../processing/materialization/persistence.py | 12 + ...utomatic_image_role_materialization_435.py | 216 ++++++++++++++++++ 5 files changed, 297 insertions(+), 24 deletions(-) create mode 100644 tests/unit/test_automatic_image_role_materialization_435.py diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index 7078339c22..394adf3046 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -23,6 +23,7 @@ from openhcs.core.component_set import ComponentSet if TYPE_CHECKING: + from openhcs.core.steps.function_output_identity import FunctionOutputIdentity from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.runtime_artifact_values import ( RuntimeValue, @@ -212,6 +213,11 @@ def materialization_base_path( del source_filename, uses_source_identity_filename, image_output_dir return Path(analysis_output_dir) / descriptor_filename + @classmethod + def retained_filename_qualifier(cls, artifact_name: str) -> str | None: + """Retain families whose existing paths already own artifact identity.""" + return None + @classmethod def normalize_runtime_payload( cls, @@ -382,6 +388,11 @@ class ImageArtifactType(ArtifactType): participates_in_main_flow_output = True carries_source_image_context = True + @classmethod + def retained_filename_qualifier(cls, artifact_name: str) -> str: + """Distinguish named retained images sharing one physical source plane.""" + return artifact_name + @classmethod def projected_materialization_base_path( cls, @@ -1051,6 +1062,19 @@ def participates_in_persistent_materialization(self) -> bool: def uses_source_identity_filename(self) -> bool: """Return whether this materialization names files by source identity.""" + @abstractmethod + def filename_qualifier(self, output_plan: ArtifactOutputPlan | None) -> str | None: + """Derive the output role required by this materialization purpose.""" + + def filename_identity_for_output( + self, + identity: FunctionOutputIdentity, + output_plan: ArtifactOutputPlan | None, + ) -> FunctionOutputIdentity: + """Apply a declared role through the shared filename identity algorithm.""" + qualifier = self.filename_qualifier(output_plan) + return identity if qualifier is None else identity.with_filename_qualifier(qualifier) + def _coerce_artifact_plan_type( plan_type: type["ArtifactPlan"], diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index 35effbb957..e91198c959 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -821,6 +821,9 @@ def record_source_descriptor( variable_components=plan.variable_components, ) if identity is not None: + identity = materialization_spec.filename_identity_for_output( + identity, output_plan, + ) try: filename = Path( FunctionOutputPathAuthority.filename_for_identity( diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index 61340a6503..c881284823 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -79,6 +79,11 @@ StreamImagePayloadMetadataProjector, StreamViewerComponentMetadataProjector, ) +from openhcs.core.steps.function_output_identity import ( + FunctionOutputIdentityAuthority, + FunctionOutputParserContext, + FunctionOutputPathAuthority, +) from openhcs.processing.materialization.constants import ( MaterializationFormat, WriteMode, @@ -2508,32 +2513,41 @@ def write_image_file( options.filename_identity is MaterializedFilenameIdentity.ARTIFACT_NAME or paths.source_identity_base_path(options) is not None ) - outputs = tuple( - ( - ( - _image_relative_output_path( - item.value, - options, - context, - sequence_index=sequence_index, + outputs = [] + for sequence_index, item in enumerate(projected_items, start=1): + if options.relative_path_template is not None: + path = _image_relative_output_path( + item.value, options, context, sequence_index=sequence_index, + ) + elif preserves_planned_path: + path = paths.primary_output_path(options) + else: + output_plan = context.output_plan + qualifier = ( + output_plan.materialization.filename_qualifier(output_plan) + if output_plan is not None and output_plan.materialization is not None + else None + ) + if qualifier is None: + filename = ( + source_stem_authority.required_source_stem(item.metadata) + + options.primary_output_suffix ) - if options.relative_path_template is not None - else ( - paths.primary_output_path(options) - if preserves_planned_path - else str( - paths.parent - / ( - source_stem_authority.required_source_stem(item.metadata) - + options.primary_output_suffix - ) - ) + else: + parser = FunctionOutputParserContext.from_processing_context( + context.context, + ).parser + identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata( + parser, item.metadata, ) - ), - item, - ) - for sequence_index, item in enumerate(projected_items, start=1) - ) + if identity is None: + raise ValueError("Retained image output has no addressable source filename identity.") + identity = output_plan.materialization.filename_identity_for_output( + replace(identity, extension=options.primary_output_suffix), output_plan, + ) + filename = FunctionOutputPathAuthority.filename_for_identity(parser, identity) + path = str(paths.parent / filename) + outputs.append((path, item)) output_paths = tuple(path for path, _item in outputs) if len(set(output_paths)) != len(output_paths): raise ValueError( @@ -4007,6 +4021,10 @@ def participates_in_runtime_export_observation(self) -> bool: """Explicit materialization specs are externally observed exports.""" return True + def filename_qualifier(self, output_plan: ArtifactOutputPlan | None) -> str | None: + """Preserve the source filenames authored by an explicit export.""" + return None + def participates_in_persistent_materialization(self) -> bool: """Explicit materialization specs write to configured persistent targets.""" return True diff --git a/openhcs/processing/materialization/persistence.py b/openhcs/processing/materialization/persistence.py index 7d976d1d09..4fd675fc20 100644 --- a/openhcs/processing/materialization/persistence.py +++ b/openhcs/processing/materialization/persistence.py @@ -1,5 +1,11 @@ """Shared persistence and export-observation policies for materialized artifacts.""" +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openhcs.core.artifacts import ArtifactOutputPlan + from .core import MaterializationSpec @@ -9,6 +15,12 @@ class TerminalMaterializationSpec(MaterializationSpec): def participates_in_runtime_export_observation(self) -> bool: return False + def filename_qualifier(self, output_plan: ArtifactOutputPlan | None) -> str | None: + """Retain the compiled image role without changing its source address.""" + if output_plan is None: + return None + return output_plan.artifact_type.retained_filename_qualifier(output_plan.name) + class StreamingOnlyMaterializationSpec(MaterializationSpec): """Prepare viewer outputs without declaring persistent or external exports.""" diff --git a/tests/unit/test_automatic_image_role_materialization_435.py b/tests/unit/test_automatic_image_role_materialization_435.py new file mode 100644 index 0000000000..f89a1a7eac --- /dev/null +++ b/tests/unit/test_automatic_image_role_materialization_435.py @@ -0,0 +1,216 @@ +"""Default retained image artifacts keep their declared output role identity.""" + +from dataclasses import replace +from pathlib import Path +import json + +import pytest +import numpy as np +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.memory import MemoryStorageBackend + +from test_function_outputs import context_stub, function_step_plan, record_output_path +from openhcs.constants.constants import Backend, Microscope, VariableComponents +from openhcs.core.aligned_image_payload import AlignedImageSliceContext, ImagePayloadBundleContext +from openhcs.core.artifacts import ArtifactOutputPlan, ArtifactSpec, ImageArtifactType +from openhcs.core.compiled_step_plan import RuntimeArtifactMaterializationPlan +from openhcs.core.pipeline.artifact_planning import ArtifactOutputMaterializationPlanner +from openhcs.core.runtime_artifact_values import RuntimeValue +from openhcs.core.runtime_image_values import ImageMetadataPayload, ImagePayloadMetadata, ImagePayloadMetadataCompositionMode +from openhcs.core.runtime_stores import RuntimeValueStore +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection +from openhcs.core.steps.function_output_identity import ( + FunctionOutputIdentityAuthority, FunctionOutputPathAuthority, +) +from openhcs.core.steps.function_outputs import ( + PrimaryImageMetadataTarget, RuntimeArtifactMaterializationAuthority, +) +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser + + +@pytest.mark.parametrize("plane_count", (1, 2)) +def test_automatic_two_role_images_publish_their_actual_saved_occurrences(tmp_path, plane_count): + filemanager = FileManager({ + Backend.DISK.value: DiskStorageBackend(), + Backend.MEMORY.value: MemoryStorageBackend(), + }) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + context.microscope_handler.microscope_type = Microscope.OPENHCS.value + context.runtime_value_store = RuntimeValueStore() + context.metadata_cache = {} + context.tiff_config = None + plan = function_step_plan( + "Declared raw and capped roles", + variable_components=((VariableComponents.Z_INDEX,) if plane_count > 1 else ()), + ) + plan.streaming_configs = {} + plan.write_backend = Backend.DISK.value + plan.output_plate_root = str(tmp_path) + plan.output_dir = tmp_path / "images" + plan.sub_dir = "images" + plan.analysis_results_dir = str(plan.output_dir) + plan.runtime_artifact_materialization = RuntimeArtifactMaterializationPlan( + persistent_enabled=True, persistent_backend=Backend.DISK.value, + ) + context.step_plans = {plan.step_index: plan} + plans = tuple( + ArtifactOutputPlan( + name=name, path=f"/memory/{name}.pkl", artifact_type=ImageArtifactType, + producer_step_index=plan.step_index, + producer_step_scope_id=plan.step_scope_id, + producer_step_name=plan.step_name, + materialization=ArtifactOutputMaterializationPlanner.materialization_for( + ArtifactSpec(name=name, artifact_type=ImageArtifactType, plan_type=ArtifactOutputPlan), (), + ), + ) + for name in ("raw_body_processing_role", "capped_outgrowth_processing_role") + ) + plan.artifact_outputs = {output.ref(): output for output in plans} + contexts = AlignedImageSliceContext.main_flow_for_output_plans(plans) + expected = {} + for output, output_context, factor in zip(plans, contexts, (1.0, 0.5), strict=True): + payloads = tuple( + ImageMetadataPayload( + (np.arange(20, dtype=np.float32).reshape(4, 5) + index) * factor, + ImagePayloadMetadata( + source_component_metadata={ + "well": "A01", "site": "1", "channel": "2", "z_index": str(index), + "timepoint": "1", "extension": ".tif", + }, + source_image_names=(output.name,), + source_voxel_spacing=SourceVoxelSpacing((1.3556, 1.3556)), + ), + ) + for index in range(1, plane_count + 1) + ) + value = ( + payloads[0] if plane_count == 1 else ImagePayloadBundleContext.from_payloads( + payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + ).compose() + ) + context.runtime_value_store.record( + RuntimeValue.normalize(output, value, axis_id="A01"), + path=output.path, backend=Backend.MEMORY.value, + ) + expected[output.name] = [] + for payload in payloads: + identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata( + context.microscope_handler.parser, payload.metadata, + ).with_filename_qualifier(output_context.output_key) + destination = plan.output_dir / FunctionOutputPathAuthority.filename_for_identity( + context.microscope_handler.parser, identity, + ) + filemanager.ensure_directory(plan.output_dir, Backend.MEMORY.value) + filemanager.ensure_directory(plan.output_dir, Backend.DISK.value) + filemanager.save(payload, str(destination), Backend.MEMORY.value) + filemanager.save(payload.data, str(destination), Backend.DISK.value) + record_output_path( + context, plan, str(destination), output_context=output_context, + image_metadata=payload.metadata, identity=identity, + ) + expected[output.name].append((destination, payload.data)) + saved = RuntimeArtifactMaterializationAuthority.materialize(context, plan) + target = replace(PrimaryImageMetadataTarget.from_plan(plan), artifact_materializations=saved) + # The original implementation raises its real duplicate projection guard + # here: both automatic materializations used the same unqualified path. + structured = target.produced_projection_metadata(context, plan) + assert {item["source_alias"] for item in structured["source_projection"]} == set(expected) + assert len(structured["source_projection"]) == 2 * plane_count + for materialized in saved: + outputs = materialized.outputs_for_backend(Backend.DISK.value) + assert len(outputs) == plane_count + for actual, (path, pixels) in zip(outputs, expected[materialized.materialization.output_plan.name], strict=True): + assert actual.path == str(path) + np.testing.assert_array_equal(tifffile.imread(path), pixels) + target.write(context, produced_plan=plan) + document = json.loads((tmp_path / "openhcs_metadata.json").read_text()) + reopened = VirtualWorkspaceSourceProjection.from_openhcs_metadata(tmp_path, document) + for name, expected_planes in expected.items(): + for path, pixels in expected_planes: + projection = reopened.source_projections_by_virtual_path[str(path.relative_to(tmp_path))] + assert projection.source_alias == name + assert projection.address.as_component_metadata()["channel"] == "2" + assert projection.image_metadata.source_voxel_spacing == SourceVoxelSpacing((1.3556, 1.3556)) + np.testing.assert_array_equal(tifffile.imread(tmp_path / projection.ref.backend_address), pixels) + + +@pytest.mark.parametrize("purpose", ( + "explicit_source", "explicit_template", "terminal_template", + "explicit_artifact", "terminal_artifact", "unnamed_terminal", +)) +def test_retained_role_policy_preserves_authored_and_unnamed_image_paths(tmp_path, purpose): + from openhcs.processing.materialization import ( + ImageFileOptions, MaterializationSpec, MaterializedFilenameIdentity, + TerminalMaterializationSpec, materialization_outputs, + ) + + options = ImageFileOptions( + filename_suffix=".tif", + relative_path_template=("authored/plane_{index}.tif" if purpose.endswith("template") else None), + filename_identity=( + MaterializedFilenameIdentity.ARTIFACT_NAME if purpose.endswith("artifact") + else MaterializedFilenameIdentity.SOURCE_IDENTITY + ), + ) + spec = ( + TerminalMaterializationSpec(options) if purpose.startswith("terminal") or purpose == "unnamed_terminal" + else MaterializationSpec(options) + ) + output_plan = None if purpose == "unnamed_terminal" else ArtifactOutputPlan( + name="DeclaredRole", path="/memory/DeclaredRole.pkl", + artifact_type=ImageArtifactType, materialization=spec, + ) + payloads = tuple( + ImageMetadataPayload(np.ones((4, 5), dtype=np.float32) * index, ImagePayloadMetadata( + source_component_metadata={ + "well": "A01", "site": "1", "channel": "2", "z_index": str(index), + "timepoint": "1", "extension": ".tif", + }, + )) + for index in (1, 2) + ) + value = ImagePayloadBundleContext.from_payloads( + payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + ).compose() + filemanager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + context = context_stub(filemanager, parser=SourceSchemaFilenameParser()) + outputs = materialization_outputs( + spec, value, str(tmp_path / "AuthoredBase"), filemanager, + context=context, output_plan=output_plan, + variable_components=(VariableComponents.Z_INDEX,), + ) + expected_paths = ( + ("authored/plane_1.tif", "authored/plane_2.tif") if purpose.endswith("template") + else ("AuthoredBase.tif",) if purpose.endswith("artifact") + else ("A01_s001_w2_z001_t001.tif", "A01_s001_w2_z002_t001.tif") + ) + assert tuple(Path(output.path).relative_to(tmp_path).as_posix() for output in outputs) == expected_paths + if len(outputs) == 2: + for actual, expected in zip(outputs, payloads, strict=True): + np.testing.assert_array_equal(actual.content, expected.data) + else: + np.testing.assert_array_equal(outputs[0].content, value.data) + + +def test_authored_duplicate_template_still_fails_before_any_save(tmp_path): + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec, materialization_outputs + + spec = TerminalMaterializationSpec(ImageFileOptions( + filename_suffix=".tif", relative_path_template="same.tif", + )) + output_plan = ArtifactOutputPlan(name="NamedRole", path="/memory/role.pkl", artifact_type=ImageArtifactType, materialization=spec) + payloads = tuple( + ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": str(index), "timepoint": "1", "extension": ".tif"}, + )) for index in (1, 2) + ) + value = ImagePayloadBundleContext.from_payloads(payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK).compose() + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + with pytest.raises(ValueError, match="produced duplicate paths"): + materialization_outputs(spec, value, str(tmp_path / "role"), manager, + context=context_stub(manager, parser=SourceSchemaFilenameParser()), + output_plan=output_plan) + assert not (tmp_path / "same.tif").exists() From 659aca3f3c5bf0645300bf2da50e4df1bb680ac3 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 11:27:19 -0400 Subject: [PATCH 072/125] docs: record latest main controls and physical collision cause for #435 --- .../shared_runtime_plumbing_384_20261001.rst | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 9b2f801d5d..8912f5518a 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1366,3 +1366,43 @@ cover; installed acceptance remains open. No original biological replay or performance claim follows from this synthetic check. Source/input/dependency, native, request/reply and failure evidence: ``/var/tmp/openhcs-derived-role-435-synthetic-acceptance-v2-20261002``. + +Current main integration and publication diagnosis, 2026-10-02 +----------------------------------------------------------- + +Main 283119c42 is normally merged after the declared independent-plane NLM +implementation and its documentation land. The combined affected NLM, +cold-feedback, agent-service, prepared geometry, native presentation and MCP +resource-lifetime controls pass 168 cases in 25.84s. Log: +``/var/tmp/openhcs-pr394-main-283119c42-integration-controls-20261002.log``. +The qualified benchmark sources and clocks above remain unchanged. + +The separately retained issue 435 public diagnostic reproduces the original +failure. Four actual projections show two correctly qualified main-flow images +but both retained artifacts address the same unqualified TIFF. The final raw +artifact overwrites the capped artifact. The same-occurrence ownership comparator +is never called: destination matching fails before that comparison. The existing +duplicate guard therefore detects a physical filename collision, not a guard +that should be relaxed. Both observer hooks call their original implementation +once; no capture errors are reported. The exact owned runtime is closed through +the public API and all SDK children are terminal. Receipt: +``/var/tmp/openhcs-derived-role-435-collision-facts-v3-20261002.json``. +An isolated repair on existing materialization-purpose and filename owners is +under review; its local test results do not yet establish public acceptance. + +The owned source-field composition prototype is also rejected: the eleven +captured valid leaves and additional mutation/constructor controls agree, but +the five context, three normalization and three identity calls save only 9.24ms +warm or 5.96ms cold. No candidate pipeline qualification or PR is opened for it. +Receipt: +``/var/tmp/openhcs-owned-source-composition-exploratory-REJECTED-20261002.json``. + +SaveImages reconstructs two source stacks solely for materialization metadata, +whose downstream consumer reads provenance. Its earlier 0.346s envelope includes +four contextualization children already measured elsewhere. After separating +those children in the retained diagnostics, the residual envelope is only about +0.10--0.114s. This is rejected as a standalone performance route; it cannot be +counted as an independent 0.346s saving. The complete source-query preimage was +not captured, so resulting metadata leaves are not a source-lookup replay. +Read-only audit: +``/var/tmp/openhcs-b764-save-images-artifact-subtree-readonly-audit-20261002.json``. From 2d756646b301382e4142a2de783432263f8ed337 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 11:37:26 -0400 Subject: [PATCH 073/125] fix: bind image filename policy to the actual rendering spec The compiled plan can declare a different materialization purpose from the spec supplied to the public render call. Bind that actual spec on MaterializationContext once and preserve the original plan, source inputs, and writer protocol. Add mismatched-purpose, projected suffix, parser-free, context-reuse, direct-writer and normalized-alias collision controls. Refs #435. --- openhcs/processing/materialization/core.py | 43 ++--- ...utomatic_image_role_materialization_435.py | 168 +++++++++++++++++- 2 files changed, 187 insertions(+), 24 deletions(-) diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index c881284823..21dd73d80c 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -81,7 +81,6 @@ ) from openhcs.core.steps.function_output_identity import ( FunctionOutputIdentityAuthority, - FunctionOutputParserContext, FunctionOutputPathAuthority, ) from openhcs.processing.materialization.constants import ( @@ -1734,6 +1733,25 @@ class MaterializationContext: source_paths: tuple[str, ...] = () pipeline_position: int | None = None output_plan: ArtifactOutputPlan | None = None + materialization_spec: MaterializationSpec | None = None + + def named_source_filename( + self, metadata: ImagePayloadMetadata, extension: str, + ) -> str | None: + """Name a retained image from the actual rendering purpose and role.""" + if self.materialization_spec is None: + return None + qualifier = self.materialization_spec.filename_qualifier(self.output_plan) + if qualifier is None: + return None + parser = SourceStemAuthoritySelection.from_processing_context(self.context).required_parser() + identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata(parser, metadata) + if identity is None: + raise ValueError("Retained image output has no addressable source filename identity.") + identity = self.materialization_spec.filename_identity_for_output( + replace(identity, extension=extension), self.output_plan, + ) + return FunctionOutputPathAuthority.filename_for_identity(parser, identity) def paths(self, options: FileOutputOptions) -> PathHelper: return PathHelper(self.base_path, options) @@ -2522,30 +2540,12 @@ def write_image_file( elif preserves_planned_path: path = paths.primary_output_path(options) else: - output_plan = context.output_plan - qualifier = ( - output_plan.materialization.filename_qualifier(output_plan) - if output_plan is not None and output_plan.materialization is not None - else None - ) - if qualifier is None: + filename = context.named_source_filename(item.metadata, options.primary_output_suffix) + if filename is None: filename = ( source_stem_authority.required_source_stem(item.metadata) + options.primary_output_suffix ) - else: - parser = FunctionOutputParserContext.from_processing_context( - context.context, - ).parser - identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata( - parser, item.metadata, - ) - if identity is None: - raise ValueError("Retained image output has no addressable source filename identity.") - identity = output_plan.materialization.filename_identity_for_output( - replace(identity, extension=options.primary_output_suffix), output_plan, - ) - filename = FunctionOutputPathAuthority.filename_for_identity(parser, identity) path = str(paths.parent / filename) outputs.append((path, item)) output_paths = tuple(path for path, _item in outputs) @@ -4186,6 +4186,7 @@ def render( *, output_path_filter: Callable[[Path], bool] | None = None, ) -> MaterializationBatch: + context = replace(context, materialization_spec=spec) groups = tuple( ( writer, diff --git a/tests/unit/test_automatic_image_role_materialization_435.py b/tests/unit/test_automatic_image_role_materialization_435.py index f89a1a7eac..58f6253db9 100644 --- a/tests/unit/test_automatic_image_role_materialization_435.py +++ b/tests/unit/test_automatic_image_role_materialization_435.py @@ -139,7 +139,8 @@ def test_automatic_two_role_images_publish_their_actual_saved_occurrences(tmp_pa @pytest.mark.parametrize("purpose", ( "explicit_source", "explicit_template", "terminal_template", - "explicit_artifact", "terminal_artifact", "unnamed_terminal", + "explicit_artifact", "terminal_artifact", "unnamed_terminal", "unnamed_explicit", + "explicit_source_declared_terminal", "terminal_source_declared_explicit", )) def test_retained_role_policy_preserves_authored_and_unnamed_image_paths(tmp_path, purpose): from openhcs.processing.materialization import ( @@ -159,9 +160,14 @@ def test_retained_role_policy_preserves_authored_and_unnamed_image_paths(tmp_pat TerminalMaterializationSpec(options) if purpose.startswith("terminal") or purpose == "unnamed_terminal" else MaterializationSpec(options) ) - output_plan = None if purpose == "unnamed_terminal" else ArtifactOutputPlan( + declared_spec = ( + TerminalMaterializationSpec(options) if purpose == "explicit_source_declared_terminal" + else MaterializationSpec(options) if purpose == "terminal_source_declared_explicit" + else spec + ) + output_plan = None if purpose in ("unnamed_terminal", "unnamed_explicit") else ArtifactOutputPlan( name="DeclaredRole", path="/memory/DeclaredRole.pkl", - artifact_type=ImageArtifactType, materialization=spec, + artifact_type=ImageArtifactType, materialization=declared_spec, ) payloads = tuple( ImageMetadataPayload(np.ones((4, 5), dtype=np.float32) * index, ImagePayloadMetadata( @@ -185,6 +191,7 @@ def test_retained_role_policy_preserves_authored_and_unnamed_image_paths(tmp_pat expected_paths = ( ("authored/plane_1.tif", "authored/plane_2.tif") if purpose.endswith("template") else ("AuthoredBase.tif",) if purpose.endswith("artifact") + else ("A01_s001_w2_z001_t001_DeclaredRole.tif", "A01_s001_w2_z002_t001_DeclaredRole.tif") if purpose == "terminal_source_declared_explicit" else ("A01_s001_w2_z001_t001.tif", "A01_s001_w2_z002_t001.tif") ) assert tuple(Path(output.path).relative_to(tmp_path).as_posix() for output in outputs) == expected_paths @@ -214,3 +221,158 @@ def test_authored_duplicate_template_still_fails_before_any_save(tmp_path): context=context_stub(manager, parser=SourceSchemaFilenameParser()), output_plan=output_plan) assert not (tmp_path / "same.tif").exists() + + +@pytest.mark.parametrize("suffix", (".tif", ".png", ".ome.tif", ".labels.tif")) +@pytest.mark.parametrize("plane_count", (1, 2)) +def test_retained_image_qualifier_respects_complete_writer_suffix(tmp_path, suffix, plane_count): + from openhcs.core.steps.function_artifact_materialization import RuntimeArtifactMaterialization + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + context = context_stub(manager, parser=SourceSchemaFilenameParser()) + plan = function_step_plan("Retained format role", variable_components=(VariableComponents.Z_INDEX,) if plane_count > 1 else ()) + plan.output_dir = tmp_path / "images" + plan.analysis_results_dir = str(plan.output_dir) + output_plan = ArtifactOutputPlan( + name="Role Name", path="/memory/role.pkl", artifact_type=ImageArtifactType, + materialization=TerminalMaterializationSpec(ImageFileOptions(filename_suffix=suffix)), + ) + payloads = tuple( + ImageMetadataPayload(np.ones((4, 5), dtype=np.uint8) * index, ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": str(index), "timepoint": "1", "extension": ".tif"}, + )) for index in range(1, plane_count + 1) + ) + value = payloads[0] if plane_count == 1 else ImagePayloadBundleContext.from_payloads( + payloads, metadata_mode=ImagePayloadMetadataCompositionMode.STACK, + ).compose() + store = RuntimeValueStore() + record = store.record(RuntimeValue.normalize(output_plan, value, axis_id="A01"), path=output_plan.path, backend=Backend.MEMORY.value) + outputs = RuntimeArtifactMaterialization.from_record( + output_plan=output_plan, record=record, plan=plan, context=context, + ).outputs(plan, context) + assert len(outputs) == plane_count + for index, (output, payload) in enumerate(zip(outputs, payloads, strict=True), 1): + identity = FunctionOutputIdentityAuthority.filename_identity_from_metadata( + context.microscope_handler.parser, payload.metadata, + ).with_filename_qualifier(output_plan.name) + expected = FunctionOutputPathAuthority.filename_for_identity( + context.microscope_handler.parser, replace(identity, extension=suffix), + ) + assert Path(output.path).name == expected + assert Path(output.path).name.endswith("_Role_Name" + suffix) + assert output.metadata.source_component_metadata["channel"] == "2" + assert output.metadata.source_component_metadata["z_index"] == str(index) + np.testing.assert_array_equal(output.content, payload.data) + + +def test_distinct_aliases_that_normalize_to_one_filename_still_conflict(tmp_path): + from openhcs.core.steps.function_artifact_materialization import RuntimeArtifactMaterialization + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + context = context_stub(manager, parser=SourceSchemaFilenameParser()) + plan = function_step_plan("Ambiguous role spelling") + plan.output_dir = tmp_path / "images" + plan.analysis_results_dir = str(plan.output_dir) + value = ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": "1", "timepoint": "1", "extension": ".tif"}, + )) + paths = [] + store = RuntimeValueStore() + for name in ("a/b", "a b"): + output_plan = ArtifactOutputPlan(name=name, path=f"/memory/{name}.pkl", artifact_type=ImageArtifactType, + materialization=TerminalMaterializationSpec(ImageFileOptions(filename_suffix=".tif"))) + record = store.record(RuntimeValue.normalize(output_plan, value, axis_id="A01"), path=output_plan.path, backend=Backend.MEMORY.value) + outputs = RuntimeArtifactMaterialization.from_record(output_plan=output_plan, record=record, plan=plan, context=context).outputs(plan, context) + paths.append(outputs[0].path) + assert paths[0] == paths[1] + # Existing main-flow destination authority rejects this declared naming + # collision instead of choosing a new suffix or inventing a channel. + from openhcs.core.steps.function_runtime import OutputPathBatchUniqueness + with pytest.raises(ValueError, match="duplicate"): + OutputPathBatchUniqueness(output_paths=paths, input_paths=[], step_name=plan.step_name, pattern_repr="declared roles").validate() + + +def test_batch_binds_actual_purpose_once_without_rewriting_plan_or_source(monkeypatch, tmp_path): + from openhcs.processing.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec + from openhcs.processing.materialization.constants import MaterializationFormat + from openhcs.processing.materialization.core import MaterializationBatch, MaterializationContext, Output, WriterSpec, _WRITERS_BY_OPTIONS + + options = ImageFileOptions(filename_suffix=".tif") + declared = TerminalMaterializationSpec(options) + explicit = MaterializationSpec(options) + plan = ArtifactOutputPlan(name="Role", path="/memory/role", artifact_type=ImageArtifactType, materialization=declared) + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + source = {"physical_source": "original"} + context = MaterializationContext( + base_path=str(tmp_path / "role"), backends=[], backend_kwargs={}, + filemanager=manager, extra_inputs=source, output_plan=plan, + materialization_spec=declared, + ) + seen = [] + def custom_writer(data, actual_options, actual_context): + assert actual_options is options + assert actual_context.output_plan is plan + assert actual_context.extra_inputs is source + seen.append(actual_context) + return [Output(path=str(tmp_path / "custom.tif"), content=data)] + monkeypatch.setitem(_WRITERS_BY_OPTIONS, ImageFileOptions, WriterSpec( + format=MaterializationFormat.IMAGE_FILE, options_type=ImageFileOptions, + write=custom_writer, primary_path=lambda outputs: outputs[0].path, + candidate_paths=lambda _options, base: (base,), + )) + pixels = np.ones((4, 5), dtype=np.float32) + first = MaterializationBatch.render(explicit, pixels, context) + second = MaterializationBatch.render(declared, pixels, first.context) + assert seen == [first.context, second.context] + assert seen[0] is first.context and seen[1] is second.context + assert first.context.materialization_spec is explicit + assert second.context.materialization_spec is declared + assert context.materialization_spec is declared + assert plan.materialization is declared + assert source == {"physical_source": "original"} + + +def test_direct_image_writer_without_purpose_binding_retains_legacy_source_path(tmp_path): + from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + from openhcs.processing.materialization.core import MaterializationContext, write_image_file + + options = ImageFileOptions(filename_suffix=".tif") + plan = ArtifactOutputPlan(name="Role", path="/memory/role", artifact_type=ImageArtifactType, + materialization=TerminalMaterializationSpec(options)) + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + value = ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_component_metadata={"well": "A01", "site": "1", "channel": "2", "z_index": "1", "timepoint": "1", "extension": ".tif"}, + )) + context = MaterializationContext(base_path=str(tmp_path / "role"), backends=[], backend_kwargs={}, + filemanager=manager, extra_inputs={}, output_plan=plan, + context=context_stub(manager, parser=SourceSchemaFilenameParser())) + outputs = write_image_file(value, options, context) + assert Path(outputs[0].path).name == "A01_s001_w2_z001_t001.tif" + assert context.materialization_spec is None + assert context.output_plan is plan + np.testing.assert_array_equal(outputs[0].content, value.data) + + +@pytest.mark.parametrize("purpose", ("named_terminal", "explicit", "unnamed_terminal")) +def test_parser_free_render_requires_parser_only_for_automatic_named_role(tmp_path, purpose): + from openhcs.processing.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec, materialization_outputs + + options = ImageFileOptions(filename_suffix=".tif") + spec = MaterializationSpec(options) if purpose == "explicit" else TerminalMaterializationSpec(options) + plan = None if purpose == "unnamed_terminal" else ArtifactOutputPlan( + name="Role", path="/memory/role", artifact_type=ImageArtifactType, materialization=spec, + ) + manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) + value = ImageMetadataPayload(np.ones((4, 5), dtype=np.float32), ImagePayloadMetadata( + source_path="/physical/OriginalSource.tif", + )) + if purpose == "named_terminal": + with pytest.raises(ValueError, match="Parser-backed source-stem authority requires a parser"): + materialization_outputs(spec, value, str(tmp_path / "Role"), manager, output_plan=plan) + else: + outputs = materialization_outputs(spec, value, str(tmp_path / "Role"), manager, output_plan=plan) + assert Path(outputs[0].path).name == "OriginalSource.tif" + np.testing.assert_array_equal(outputs[0].content, value.data) + assert value.metadata.source_path == "/physical/OriginalSource.tif" From df9466c9154650385aadad587dac790fc9e4b2ce Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 11:48:38 -0400 Subject: [PATCH 074/125] docs: record #435 actual-purpose repair and public execution gates --- .../shared_runtime_plumbing_384_20261001.rst | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 8912f5518a..6c9f01662f 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1406,3 +1406,56 @@ counted as an independent 0.346s saving. The complete source-query preimage was not captured, so resulting metadata leaves are not a source-lookup replay. Read-only audit: ``/var/tmp/openhcs-b764-save-images-artifact-subtree-readonly-audit-20261002.json``. + +Automatic image-role repair and public execution, 2026-10-02 +---------------------------------------------------------- + +Commits 0b6c66560 and 2d756646b repair the physical issue 435 collision on +existing owners. Artifact type and materialization purpose supply the retained +role; shared filename identity supplies parser-backed names. MaterializationBatch +binds the original rendering spec onto its existing context, because a public +caller may render a spec different from the compiled plan's default. The original +output plan and source references remain intact. Explicit exports keep their +authored paths. Both scalar and projected retained images preserve physical +coordinates and complete filename suffixes. Missing parsers use the existing +required-parser error policy. Strict projection and path-conflict guards remain +unchanged. + +The initial candidate's two actual-purpose mismatch controls fail before this +binding repair. The initial scoped R0 also reports a foreign optional-plan probe; +the corrected owner eliminates that probe without changing the guard. Original +scoped R0 and R1 pass for the complete repair versus 51533665e. Both original +failures are retained. Related controls pass 171 cases; after normal main +283119c42 integration at 053c9d445, 183 affected controls pass. An attempted +unrelated Torch NLM control stops at collection because Torch is absent; that +failure remains separate and no dependency changes follow. Handoff: +``/var/tmp/openhcs-automatic-image-role-435-reviewed-handoff-20261002.json``. + +The clean frozen 053c9d445 source completes public MCP startup preparation, +compilation and all five original processing steps. The output contains exactly +the two distinct qualified role TIFFs and two matching source-projection rows. +The fourth journey then stops at its inspection assertion: the API reports +PARTIAL with the sole missing-grid warning, because the synthetic declaration +does not supply plate grid dimensions. This original whole-journey RED is +retained; it is not a pixel-comparison failure or a completed fresh-process +readback acceptance. The runtime and all SDK children are terminal. Receipt: +``/var/tmp/openhcs-derived-role-435-synthetic-acceptance-v4-20261002/journey.json``. +Supplemental public inventory, complete pixel and fresh-process checks remain +pending on these unchanged outputs. + +The reviewed source is normally integrated into PR394 at fa694eb56. Its affected +current NLM, cold-feedback, agent-service, prepared geometry, native presentation, +MCP lifetime and new role controls pass 194 cases in 20.43s. Log: +``/var/tmp/openhcs-pr394-automatic-role-435-current-integration-controls-20261002.log``. +Neither this bug repair nor its public synthetic run establishes a performance +gain. Accepted benchmark clocks remain pinned to main4754 and candidateb764. + +The global runtime reassessment finds no sufficient measured optimization route +yet. Removing 485 provenance births and 485 identity births in captured context +leaves already saved only 4.82ms and changed malformed-input ordering. The next +candidate must remove whole repeated projection/adapter transactions, rather +than another constructor or field-query leaf. The strongest remaining joint +frontier is 3.0201s with incomplete whole-lane replay coverage; counts from the +instrumented profile cannot establish its reducible fraction. Audit and smallest +missing capture recipe: +``/var/tmp/openhcs-b764-whole-lifecycle-architecture-reassessment-20261002.json``. From 185efca6267117b0ea5966e51dcfeb2c5edbb0e3 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 11:53:05 -0400 Subject: [PATCH 075/125] docs: qualify #435 role pixels and fresh public readback --- .../shared_runtime_plumbing_384_20261001.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 6c9f01662f..5c9091bcda 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1459,3 +1459,20 @@ frontier is 3.0201s with incomplete whole-lane replay coverage; counts from the instrumented profile cannot establish its reducible fraction. Audit and smallest missing capture recipe: ``/var/tmp/openhcs-b764-whole-lifecycle-architecture-reassessment-20261002.json``. + +Issue 435 supplemental public readback passes on the unchanged V4 output. Two +fresh SDK processes expose identical complete inventory, sample and projection +facts. Both 1024-by-1024 float32 role images pass the unchanged existing CP pixel +comparison with zero out-of-tolerance pixels: raw maximum difference is zero; +capped maximum difference is 1.1920928955078125e-7 under atol=rtol=1e-6. Both +roles retain physical C2 and calibration within the existing inspection spacing +tolerance of 1e-12. The original controller, journey, inputs, source, dependencies, +native binaries and all physical output hashes remain unchanged, and both reader +children exit zero. The original V4 journey remains RED. The supplemental +inspection check follows the existing status policy, retaining PARTIAL for the +sole undeclared-grid warning without fabricating grid metadata; spacing uses the +existing inspection tolerance rather than literal equality after TIFF formatting. +This is synthetic source qualification, not original biological or installed-root +acceptance. Supplemental receipt: +``/var/tmp/openhcs-derived-role-435-v4-readback-supplement-20261002/receipt.json`` +(SHA256 4f0f0550e50aaff297d134ee5accf2b2da01297305394a155fe79e827b844fdf). From 6f257f10e9047f1bf4d3e3cdc52f5f90c03ee1ff Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 12:35:23 -0400 Subject: [PATCH 076/125] docs: record current runtime qualification and workload priority --- .../shared_runtime_plumbing_384_20261001.rst | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 5c9091bcda..effb31e7ac 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1476,3 +1476,44 @@ This is synthetic source qualification, not original biological or installed-roo acceptance. Supplemental receipt: ``/var/tmp/openhcs-derived-role-435-v4-readback-supplement-20261002/receipt.json`` (SHA256 4f0f0550e50aaff297d134ee5accf2b2da01297305394a155fe79e827b844fdf). + +Current source qualification and workload priority, 2026-10-02 +------------------------------------------------------------ + +Main 142196857 (callable documentation ownership) is normally merged at +9aa3918d2. The affected declared-documentation, agent-service, cold-feedback, +automatic-image-role and plane-NLM integration controls pass 170 cases in +14.77s. Log: +``/var/tmp/openhcs-pr394-main142-documentation-role-integration-controls-20261002.log``. + +A clean frozen 9aa3918d2 ordinary public-driver observation completes one CPU5 +inline worker/thread with compilation 2.103262s, execution 7.870635s and total +10.781240s. Server/library/kernel startup and shutdown remain outside pipeline +clocks. All six CSVs, 120 TIFFs, 180 disk projection references and three source +volume hashes agree exactly with the retained qualified reference and b764 +output; source, dependencies, interpreter and native binaries remain unchanged. +Native science is transitive through that exact reference. This single current +observation does not replace the earlier repeated ABBA comparison or establish +a new optimization gain. Receipt: +``/var/tmp/openhcs-pr394-role-qualified-3d-science-20261002.json``. + +The descriptive comparison against the two retained warm native 3D observations +(mean 14.395949s) is 1.829x execution and 1.335x total. 3D is weakest among the +three qualified representatives (3D, Advanced and IFC), whose OpenHCS revisions +differ. The global weakest case remains unknown: all30 cached native clocks +include subprocess startup and the older multicore CP scaling baseline is +projected. Historical WoundHealing, TrackObjects and worm cases require current +matched warm-native qualification before ranking the whole catalog. Receipt: +``/var/tmp/openhcs-current-native-workload-priority-revalidation-20261002.json``. + +The next whole-transaction diagnostic is rejected before launch when its saved +representative identity pairs predict 2,719,141,800 bytes for three 60-request +cohorts alone, exceeding the transport's 1GiB root budget. No pipeline is run +for V4 and none of its frozen files are changed. This prediction is not an +observed complete production roster. A saved grouped-graph experiment reduces +those three representative graphs to 81,450,818 bytes while preserving actual +saved payloads; cache subsets contain no arrays. A separate V5 preparation +must capture actual ordered requests and detached pre/post cache states before +its replay or a performance candidate can be admitted. Receipts: +``/var/tmp/openhcs-whole-cohort-v4-budget-preflight-REJECTED-20261002.json`` and +``/var/tmp/openhcs-grouped-identity-budget-saved-experiment-v3-20261002/receipt.json``. From d99b2eb4c96bf2b36c206ae256c873433b62c0a0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 13:33:45 -0400 Subject: [PATCH 077/125] docs: qualify warm native slow cases and redirect runtime priority --- .../shared_runtime_plumbing_384_20261001.rst | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index effb31e7ac..d8a3c6bfb9 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1517,3 +1517,103 @@ must capture actual ordered requests and detached pre/post cache states before its replay or a performance candidate can be admitted. Receipts: ``/var/tmp/openhcs-whole-cohort-v4-budget-preflight-REJECTED-20261002.json`` and ``/var/tmp/openhcs-grouped-identity-budget-saved-experiment-v3-20261002/receipt.json``. + +Fresh native qualification changes the primary target +--------------------------------------------------- + +Four additional ordinary current9aa cases now have matched source inventories +and actual warm native runs. Each native case excludes one warmup and retains +both measured repetitions. OpenHCS uses one CPU5 inline worker/thread, mandatory +READY warmup before compilation, the default OUTCOMES observer and memory policy. +Server startup and shutdown are excluded. The following ratios are descriptive: +one OpenHCS observation against the mean of two native observations, not a +repeated A/B optimization claim. Native time includes preparation, modules, +post-run and measurement closure, excluding imports, JVM startup and pipeline +loading; OpenHCS total includes compilation. + +.. list-table:: Current single-worker observations, seconds + :header-rows: 1 + + * - Case + - OpenHCS execution + - OpenHCS total + - Native mean + - Execution speedup + - Total speedup + * - WoundHealing + - 3.571737 + - 4.096850 + - 3.779188 + - 1.058x + - 0.922x + * - TrackObjects + - 5.036489 + - 6.400329 + - 8.276148 + - 1.643x + - 1.293x + * - UntangleWorms + - 2.194352 + - 3.206172 + - 2.974680 + - 1.356x + - 0.928x + * - UntangleWormsBrightField + - 3.064656 + - 4.264850 + - 6.152021 + - 2.007x + - 1.443x + +All eight comparisons pass the existing measurement/schema/discrete gate and +unchanged numeric tolerances. WoundHealing and BrightField also pass complete +saved-output science. TrackObjects and UntangleWorms fail saved-image content +comparison in both repetitions; their ratios therefore have measurement-only +qualification. Physical output inventories match. All original RED receipts +remain unchanged. TrackObjects saves 21 three-panel PNGs: original-image and +outline panels agree exactly, while the tracked-object panels are gray instead +of the native colored labels and white IDs. The current callable deletes the +requested saved-image options and returns its unchanged input image. Worm +images have matching backgrounds and colors but 96 differing outline pixels +across two outputs. Sparse-overlap rendering is a candidate cause requiring +actual producer inputs, not a proved segmentation diagnosis. + +WoundHealing is weakest among seven retained representative measurement-science +scopes, replacing 3D as the primary investigation. This is not a full-catalog +ranking. WoundHealing needs about 1.682142s execution reduction and 2.207256s +total reduction to reach twice its native throughput. Its native comparison +uses exactly two selected 2304-by-1536 RGB JPGs; TrackObjects uses all 21 selected +sequence frames, and both worm cases retain their actual source bindings and +training inputs. Source, dependencies, interpreter, native binaries and all +original inputs are frozen before and after. Evidence: +``/var/tmp/openhcs-current-slowcase-priority-and-png-audit-20261002.json`` +(SHA256 c84d5c0f39bf2a246e00eb7cbb42a7457849efb6a974a6d78fee0e68c6c270ef). + +The existing worker-owned profile on current9aa WoundHealing preserves exact +ordinary CSV outputs. It attributes 0.604904s to 22 runtime stack operations, +0.551025s to 16 image normalizations, 0.288266s to two centroid queries and +0.464524s to two color reductions. These profile durations are diagnostic costs, +not measured savings. Metadata composition itself is only 0.016512s. Source +recording's 0.251105s overlaps normalization and must not be added to it. Even +eliminating stacking, normalization and centroid costs entirely gives an +optimistic 1.444195s, below the execution gap before mandatory copies. Another +small metadata leaf is rejected as a sufficient route. Receipt: +``/var/tmp/openhcs-current9aa-wound-profile-dominant-frontier-20261002.json``. + +The bounded actual-input normalization capture completes successfully: all 16 +calls are recorded, with two unchanged V6 before/after graph pairs for the +large integer arrivals. Both consume the same two-frame uint8 RGB stack, +shape (2, 1536, 2304, 3), scale 255 and float32 target. Original input pixels +and public metadata remain unchanged; source/environment/native freezes pass. +Capture clocks are not performance evidence. Independent whole-output science +and original-leaf replay remain separate gates. Receipt: +``/var/tmp/openhcs-wound-intensity-leaf-capture-v1-20261002/observations.json``. +The generic consumer investigation must preserve exact compiled artifact +selection, source projection and stack-broadcast laws; raw integer pixels cannot +be relabeled as normalized unit-interval pixels to avoid a conversion. + +The former 3D grouped-cohort V5 diagnostic remains prepared and unlaunched after +this priority change. No rejected route is promoted because its preparation is +already complete. The tracking renderer is being investigated in an independent +main142 worktree with actual 21-frame producer inputs and exact native panel +replay; no saved-image repair is included or claimed here yet. From e8dd23fdbf2de8532569f58bef98c0aadbd11647 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 15:23:30 -0400 Subject: [PATCH 078/125] perf: share dense object coordinate reduction across storage and CP backends Dense int32 storage reduces coordinates directly instead of materializing sparse pixels. Tracking and relationships use the same numerical reducer with their existing 2D ABI, and declared preparation readies mutable and read-only storage signatures before execution. Refs #384. Exact saved Wound and Track numerical/ownership replay passes; this is a local coordinate-lane gain pending ordinary pipeline qualification. --- openhcs/core/runtime_object_labels.py | 87 ++++++++++ .../backends/cellprofiler/_backend.py | 2 + .../backends/cellprofiler/_preparation.py | 2 + .../backends/cellprofiler/relationships.py | 32 +--- .../backends/cellprofiler/tracking.py | 30 +--- .../test_dense_object_coordinate_reduction.py | 163 ++++++++++++++++++ 6 files changed, 263 insertions(+), 53 deletions(-) create mode 100644 tests/unit/test_dense_object_coordinate_reduction.py diff --git a/openhcs/core/runtime_object_labels.py b/openhcs/core/runtime_object_labels.py index 056dc95c89..fac841b9bb 100644 --- a/openhcs/core/runtime_object_labels.py +++ b/openhcs/core/runtime_object_labels.py @@ -14,6 +14,7 @@ import numpy as np from metaclass_registry import AutoRegisterMeta +from numba import njit from openhcs.core import ( runtime_array_values, @@ -1198,6 +1199,38 @@ def dense_data( def sparse_ijv_rows(self, labels: object) -> SparseIJVLabelRows: return SparseIJVLabelRows.from_dense_stack(cast(np.ndarray, labels)) + @classmethod + def prepare_coordinates(cls) -> None: + """Prepare both mutability signatures of the dense coordinate reducer.""" + labels = np.array([[0, 1]], dtype=np.int32) + for writeable in (True, False): + labels.flags.writeable = writeable + _dense_label_coordinate_centers_numba(labels.ravel(), 1, 2, 1, 2) + dense_label_centers_2d_numba(labels, 1) + + def axis_centers( + self, labels: object, *, domain: Sequence[int] + ) -> tuple[tuple[np.ndarray, ...], np.ndarray]: + if ( + type(labels) is not np.ndarray + or labels.dtype != np.dtype(np.int32) + or labels.ndim not in (2, 3) + or any(size > np.iinfo(np.int32).max for size in labels.shape) + or type(domain) is not tuple + or any(type(value) is not int for value in domain) + or max(domain, default=0) >= np.iinfo(np.intp).max + ): + return super().axis_centers(labels, domain=domain) + maximum_label = max(int(labels.max(initial=0)), max(domain, default=0)) + centers, counts = _dense_label_coordinate_centers_numba( + labels.ravel(order="C"), + labels.shape[-2], labels.shape[-1], maximum_label, labels.ndim, + ) + return ( + tuple(centers[:, axis].copy() for axis in range(labels.ndim)), + counts.copy(), + ) + def stack_planes( self, labels: Sequence[object], @@ -1399,6 +1432,14 @@ def sparse_ijv_rows(self, labels: object) -> SparseIJVLabelRows: label_data ) + def axis_centers( + self, labels: object, *, domain: Sequence[int] + ) -> tuple[tuple[np.ndarray, ...], np.ndarray]: + label_data = self.label_data(labels) + return ObjectLabelStorageStrategy.for_value(label_data).axis_centers( + label_data, domain=domain + ) + def stack_planes( self, labels: Sequence[object], @@ -1551,6 +1592,52 @@ def object_label_axis_centers( ) +@njit(cache=True) +def _dense_label_coordinate_centers_numba( + flat_labels: np.ndarray, + height: int, + width: int, + maximum_label: int, + coordinate_count: int, +) -> tuple[np.ndarray, np.ndarray]: + """Reduce dense positive labels once in their declared row-major geometry.""" + sums = np.zeros((maximum_label + 1, 3), dtype=np.float64) + counts = np.zeros(maximum_label + 1, dtype=np.int64) + plane_size = height * width + plane_count = 0 if plane_size == 0 else flat_labels.size // plane_size + offset = 0 + for plane in range(plane_count): + for y in range(height): + for x in range(width): + label_id = int(flat_labels[offset]) + offset += 1 + if label_id > 0 and label_id <= maximum_label: + sums[label_id, 0] += plane + sums[label_id, 1] += y + sums[label_id, 2] += x + counts[label_id] += 1 + centers = np.empty((maximum_label + 1, coordinate_count), dtype=np.float64) + for label_id in range(maximum_label + 1): + for axis in range(coordinate_count): + centers[label_id, axis] = ( + np.nan if counts[label_id] == 0 + else sums[label_id, axis + 3 - coordinate_count] / counts[label_id] + ) + return centers, counts + + +@njit(cache=True) +def dense_label_centers_2d_numba( + labels: np.ndarray, label_count: int +) -> np.ndarray: + """Validate compiled two-dimensional geometry and return its y/x centers.""" + height, width = labels.shape + centers, _counts = _dense_label_coordinate_centers_numba( + labels.ravel(), height, width, label_count, 2 + ) + return centers + + def object_label_storage_is_sparse_ijv(payload: object) -> bool: """Return whether the nominal storage authority owns sparse-IJV rows.""" authority = ObjectLabelStorageStrategy.for_value(payload) diff --git a/openhcs/processing/backends/cellprofiler/_backend.py b/openhcs/processing/backends/cellprofiler/_backend.py index e37742d5b1..811a6deebc 100644 --- a/openhcs/processing/backends/cellprofiler/_backend.py +++ b/openhcs/processing/backends/cellprofiler/_backend.py @@ -14,6 +14,7 @@ from openhcs.constants.constants import MemoryType from openhcs.core.callable_contract import CallableContract from openhcs.core.processing_preparation import PersistentNumbaKernelPreparation +from openhcs.core.runtime_object_labels import DenseArrayObjectLabelStorageStrategy from openhcs.core.runtime_plane_projection import RuntimeSliceInvariantValue @@ -306,6 +307,7 @@ def prepare_registered_family(cls) -> None: """Prepare every registered backend implementation for compiler warmup.""" if cls is CellProfilerBackendStrategyMixin: return + DenseArrayObjectLabelStorageStrategy.prepare_coordinates() snapshot = CellProfilerBackendRegistrySnapshot.for_family( cls, MemoryType.NUMPY, diff --git a/openhcs/processing/backends/cellprofiler/_preparation.py b/openhcs/processing/backends/cellprofiler/_preparation.py index 933a8a3795..cea2850ff1 100644 --- a/openhcs/processing/backends/cellprofiler/_preparation.py +++ b/openhcs/processing/backends/cellprofiler/_preparation.py @@ -3,6 +3,7 @@ from __future__ import annotations from openhcs.core.processing_preparation import RegisteredNumbaKernelPreparation +from openhcs.core.runtime_object_labels import DenseArrayObjectLabelStorageStrategy from openhcs.processing.backends.cellprofiler.perf_fixtures import capture_enabled @@ -20,4 +21,5 @@ def can_prepare_in_child(cls) -> bool: def prepare_registered_family(cls) -> None: if capture_enabled(): return + DenseArrayObjectLabelStorageStrategy.prepare_coordinates() super().prepare_registered_family() diff --git a/openhcs/processing/backends/cellprofiler/relationships.py b/openhcs/processing/backends/cellprofiler/relationships.py index a775ecd7c2..b69c31d2f6 100644 --- a/openhcs/processing/backends/cellprofiler/relationships.py +++ b/openhcs/processing/backends/cellprofiler/relationships.py @@ -865,6 +865,7 @@ def aggregate_child_feature_name( ) from openhcs.core.runtime_tabular_values import ColumnarRows from openhcs.core.runtime_object_labels import ( + dense_label_centers_2d_numba, ObjectLabelValue, object_label_value_with_dense_labels, ) @@ -1854,7 +1855,7 @@ def label_centers(self, labels: np.ndarray) -> np.ndarray: label_count = int(labels.max()) if label_count == 0: return np.empty((0, 2), dtype=np.float64) - centroids = _label_centroids_numba(np.ascontiguousarray(labels), label_count) + centroids = dense_label_centers_2d_numba(np.ascontiguousarray(labels), label_count) return centroids[1:] @@ -2193,29 +2194,6 @@ def _parents_of_from_overlap_counts_numba( return parents_of -@njit(cache=True) -def _label_centroids_numba(labels: np.ndarray, label_count: int) -> np.ndarray: - sums = np.zeros((label_count + 1, 2), dtype=np.float64) - counts = np.zeros(label_count + 1, dtype=np.int64) - height, width = labels.shape - for row in range(height): - for col in range(width): - label_id = int(labels[row, col]) - if label_id > 0 and label_id <= label_count: - sums[label_id, 0] += row - sums[label_id, 1] += col - counts[label_id] += 1 - centroids = np.empty((label_count + 1, 2), dtype=np.float64) - for label_id in range(label_count + 1): - if counts[label_id] == 0: - centroids[label_id, 0] = np.nan - centroids[label_id, 1] = np.nan - else: - centroids[label_id, 0] = sums[label_id, 0] / counts[label_id] - centroids[label_id, 1] = sums[label_id, 1] / counts[label_id] - return centroids - - @njit(cache=True) def _calculate_centroid_distances_numba( parent_labels: np.ndarray, @@ -2229,8 +2207,8 @@ def _calculate_centroid_distances_numba( distances[child_idx] = np.nan if child_count == 0 or parent_count == 0: return distances - parent_centroids = _label_centroids_numba(parent_labels, parent_count) - child_centroids = _label_centroids_numba(child_labels, child_count) + parent_centroids = dense_label_centers_2d_numba(parent_labels, parent_count) + child_centroids = dense_label_centers_2d_numba(child_labels, child_count) for child_idx in range(child_count): parent_id = int(parents_of[child_idx]) child_id = child_idx + 1 @@ -2282,7 +2260,7 @@ def _calculate_minimum_distances_numba( distances[child_idx] = np.nan if child_count == 0 or parent_count == 0: return distances - child_centroids = _label_centroids_numba(child_labels, child_count) + child_centroids = dense_label_centers_2d_numba(child_labels, child_count) height, width = parent_labels.shape counts = np.zeros(parent_count + 1, dtype=np.int64) for row in range(height): diff --git a/openhcs/processing/backends/cellprofiler/tracking.py b/openhcs/processing/backends/cellprofiler/tracking.py index 3a36e51e4e..e591ba0d42 100644 --- a/openhcs/processing/backends/cellprofiler/tracking.py +++ b/openhcs/processing/backends/cellprofiler/tracking.py @@ -39,6 +39,7 @@ from openhcs.core.registry_strategies import EnumKeyedStrategyMixin from openhcs.core.runtime_object_labels import ( ObjectLabelValue, + dense_label_centers_2d_numba, object_label_dense_array, ) from openhcs.core.runtime_relationships import ObjectRelationshipDeclaration @@ -830,7 +831,7 @@ def label_centers(self, labels: np.ndarray) -> tuple[np.ndarray, np.ndarray]: label_count = int(labels_array.max()) if labels_array.size else 0 if label_count == 0: return (np.array([], dtype=np.float64), np.array([], dtype=np.float64)) - centers = _label_centers_numba(np.ascontiguousarray(labels_array), label_count) + centers = dense_label_centers_2d_numba(np.ascontiguousarray(labels_array), label_count) return (centers[1:, 0], centers[1:, 1]) def track_by_overlap( @@ -1461,29 +1462,6 @@ def _apply_final_age_measurements(frame_results: TrackingFrameResults) -> None: final_age_records[slice_index, object_label].final_age = int(lifetime) -@njit(cache=True) -def _label_centers_numba(labels: np.ndarray, label_count: int) -> np.ndarray: - sums = np.zeros((label_count + 1, 2), dtype=np.float64) - counts = np.zeros(label_count + 1, dtype=np.int64) - height, width = labels.shape - for y in range(height): - for x in range(width): - label_id = int(labels[y, x]) - if label_id > 0 and label_id <= label_count: - sums[label_id, 0] += y - sums[label_id, 1] += x - counts[label_id] += 1 - centers = np.empty((label_count + 1, 2), dtype=np.float64) - for label_id in range(label_count + 1): - if counts[label_id] == 0: - centers[label_id, 0] = np.nan - centers[label_id, 1] = np.nan - else: - centers[label_id, 0] = sums[label_id, 0] / counts[label_id] - centers[label_id, 1] = sums[label_id, 1] / counts[label_id] - return centers - - @njit(cache=True) def _track_by_overlap_numba( frame: TrackingKernelFrame, @@ -1528,8 +1506,8 @@ def _track_by_overlap_numba( def _track_by_distance_numba( frame: TrackingKernelFrame, pixel_radius: int ) -> tuple[np.ndarray, np.ndarray, np.ndarray, int]: - current_centers = _label_centers_numba(frame.current_labels, frame.current_count) - old_centers = _label_centers_numba(frame.old_labels, frame.old_count) + current_centers = dense_label_centers_2d_numba(frame.current_labels, frame.current_count) + old_centers = dense_label_centers_2d_numba(frame.old_labels, frame.old_count) new_labels = np.zeros(frame.current_count, dtype=np.int64) parent_object_numbers = np.zeros(frame.current_count, dtype=np.int64) parent_image_numbers = np.zeros(frame.current_count, dtype=np.int64) diff --git a/tests/unit/test_dense_object_coordinate_reduction.py b/tests/unit/test_dense_object_coordinate_reduction.py new file mode 100644 index 0000000000..b62bd89290 --- /dev/null +++ b/tests/unit/test_dense_object_coordinate_reduction.py @@ -0,0 +1,163 @@ +"""Storage reduction preserves its generic oracle and compiled backend domains.""" + +import os +import subprocess +import sys +import textwrap +import warnings +from pathlib import Path + +import numpy as np +import pytest + +from openhcs.core.runtime_object_labels import ( + ObjectLabelPayload, + ObjectLabelStorageStrategy, + ObjectLabelVariantData, + object_label_axis_centers, +) +from openhcs.core.runtime_sparse_labels import SparseIJVLabelRows + + +def generic_centers(labels, domain): + return ObjectLabelStorageStrategy.axis_centers( + ObjectLabelStorageStrategy.for_value(labels), labels, domain=domain + ) + + +def assert_centers(actual, expected): + for actual_array, expected_array in zip( + (*actual[0], actual[1]), (*expected[0], expected[1]), strict=True + ): + np.testing.assert_array_equal(actual_array, expected_array) + assert actual_array.tobytes() == expected_array.tobytes() + assert actual_array.dtype == expected_array.dtype + assert actual_array.strides == expected_array.strides + assert actual_array.flags.owndata == expected_array.flags.owndata + assert actual_array.base is None + assert not np.shares_memory(*actual[0][:2]) + + +@pytest.mark.parametrize("wrapped", (False, True)) +@pytest.mark.parametrize("shape", ((4, 5), (3, 4, 5))) +def test_dense_storage_reduces_without_sparse_materialization(monkeypatch, wrapped, shape): + labels = np.zeros(shape, dtype=np.int32) + labels[..., 1:3, 2:4] = 1 + labels[..., 3, 0] = 4 + domain = (1, 2, 4, 7) + expected = generic_centers(labels, domain) + value = ( + ObjectLabelPayload(variant_data=ObjectLabelVariantData(labels=labels)) + if wrapped else labels + ) + before = labels.copy() + + def reject_materialization(_cls, _labels): + raise AssertionError("Coordinate reduction materialized sparse pixels") + + monkeypatch.setattr(SparseIJVLabelRows, "from_dense_stack", classmethod(reject_materialization)) + assert_centers(object_label_axis_centers(value, domain=domain), expected) + np.testing.assert_array_equal(labels, before) + + +@pytest.mark.parametrize("layout", ("c", "f", "view", "readonly", "empty")) +def test_layout_and_mutable_pixels_preserve_generic_result(layout): + labels = np.arange(30, dtype=np.int32).reshape(5, 6) % 5 + if layout == "f": + labels = np.asfortranarray(labels) + elif layout == "view": + labels = labels[::-1, ::2] + elif layout == "readonly": + labels.flags.writeable = False + elif layout == "empty": + labels = np.zeros((2, 0, 3), dtype=np.int32) + domain = (1, 2, 4, 9) + assert_centers(object_label_axis_centers(labels, domain=domain), generic_centers(labels, domain)) + if labels.size and labels.flags.writeable: + labels.flat[0] = 4 + assert_centers(object_label_axis_centers(labels, domain=domain), generic_centers(labels, domain)) + + +@pytest.mark.parametrize("dtype", (np.bool_, np.float64, np.int64, np.uint64)) +def test_other_dtype_conversion_and_warning_policy_stays_generic(dtype): + labels = np.array([[0, 1], [2, 3]], dtype=dtype) + if dtype is np.float64: + labels[0] = (0.5, np.nan) + elif dtype is np.uint64: + labels[0, 1] = 2**32 + 1 + outcomes = [] + for call in (generic_centers, lambda x, domain: object_label_axis_centers(x, domain=domain)): + with warnings.catch_warnings(record=True) as observed: + warnings.simplefilter("always") + try: + result = call(labels, (1, 2, 3, 8)) + except ValueError as error: + result = (type(error), str(error)) + outcomes.append((result, [(type(w.message), str(w.message)) for w in observed])) + if dtype is np.uint64: + assert outcomes[0][0] == outcomes[1][0] + else: + assert_centers(outcomes[0][0], outcomes[1][0]) + assert outcomes[0][1] == outcomes[1][1] + + +def test_sparse_overlap_and_subtype_keep_existing_representation(): + sparse = SparseIJVLabelRows(np.array([[0, 0, 1], [0, 0, 2], [2, 3, 1]], dtype=np.int32)) + assert_centers(object_label_axis_centers(sparse, domain=(1, 2, 4)), generic_centers(sparse, (1, 2, 4))) + + class DenseSubtype(np.ndarray): + @property + def dtype(self): + raise AssertionError("Subtype properties must not be probed before generic coercion") + + subtype = np.array([[1, 0], [2, 2]], dtype=np.int32).view(DenseSubtype) + assert_centers(object_label_axis_centers(subtype, domain=(1, 2, 4)), generic_centers(subtype, (1, 2, 4))) + + +@pytest.mark.parametrize("shape", ((), (4,), (2, 3, 4, 5))) +def test_invalid_shape_keeps_original_error(shape): + labels = np.zeros(shape, dtype=np.int32) + with pytest.raises(ValueError) as original: + generic_centers(labels, (1,)) + with pytest.raises(type(original.value), match="SparseIJVLabelRows.from_dense_stack") as current: + object_label_axis_centers(labels, domain=(1,)) + assert str(current.value) == str(original.value) + + +def test_primary_objects_preparation_readies_all_admitted_storage_signatures(tmp_path): + script = textwrap.dedent(""" + from unittest.mock import patch + import numpy as np + from openhcs.core.callable_contract import prepare_processing_callable + from openhcs.core.runtime_object_labels import ( + _dense_label_coordinate_centers_numba, dense_label_centers_2d_numba, + object_label_axis_centers, + ) + from openhcs.processing.backends.cellprofiler.primary_objects import identify_primary_objects + assert not _dense_label_coordinate_centers_numba.signatures + prepare_processing_callable(identify_primary_objects) + kernels = (_dense_label_coordinate_centers_numba, dense_label_centers_2d_numba) + signatures = tuple(tuple(kernel.signatures) for kernel in kernels) + assert all(signatures) + + def reject_compilation(signature): + raise AssertionError(('Late coordinate compilation', str(signature))) + + with patch.object(_dense_label_coordinate_centers_numba, 'compile', + side_effect=reject_compilation), patch.object( + dense_label_centers_2d_numba, 'compile', side_effect=reject_compilation): + for shape in ((4, 5), (3, 4, 5)): + base = np.arange(np.prod(shape), dtype=np.int32).reshape(shape) % 4 + for labels in (base, np.asfortranarray(base), base[..., ::-1], base.copy()): + object_label_axis_centers(labels, domain=(1, 2, 3, 6)) + labels.flags.writeable = False + object_label_axis_centers(labels, domain=(1, 2, 3, 6)) + assert tuple(tuple(kernel.signatures) for kernel in kernels) == signatures + """) + environment = os.environ.copy() + environment.update(OPENHCS_CPU_ONLY="true", NUMBA_CACHE_DIR=str(tmp_path / "kernels")) + result = subprocess.run( + (sys.executable, "-c", script), cwd=Path(__file__).parents[2], + env=environment, capture_output=True, text=True, timeout=120, + ) + assert result.returncode == 0, result.stdout + result.stderr From 49205ac42694387f9de078aeba89c67a0f53b14d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 15:23:51 -0400 Subject: [PATCH 079/125] fix: prewarm masked polynomial illumination kernels Prepare masked and unmasked polynomial signatures on the existing callable preparation hook, including readonly canonical inputs. Readiness controls refuse any subsequent dispatcher compilation or cache load while validating exact fitting behavior, input isolation and geometry errors. Refs #384. Intended for PR #394. --- .../backends/cellprofiler/illumination.py | 10 +++ .../test_cellprofiler_kernel_preparation.py | 80 +++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/openhcs/processing/backends/cellprofiler/illumination.py b/openhcs/processing/backends/cellprofiler/illumination.py index d86e17def8..1dab00b4e5 100644 --- a/openhcs/processing/backends/cellprofiler/illumination.py +++ b/openhcs/processing/backends/cellprofiler/illumination.py @@ -1003,6 +1003,16 @@ def _prepare_correct_illumination_calculate() -> None: filter_size_method=FilterSizeMethod.AUTOMATIC, rescale_option=RescaleOption.YES, ) + # Polynomial fitting canonicalizes dtype/layout, but retains read-only + # contiguous arrays. Prepare masked and unmasked mutability signatures. + polynomial_image = np.ascontiguousarray(image, dtype=np.float64) + polynomial_mask = np.ones(image.shape, dtype=np.bool_) + for image_writeable in (True, False): + polynomial_image.flags.writeable = image_writeable + fit_polynomial_surface(polynomial_image, None) + for mask_writeable in (True, False): + polynomial_mask.flags.writeable = mask_writeable + fit_polynomial_surface(polynomial_image, polynomial_mask) convex_hull_image = np.linspace(0.0, 1.0, 32 * 32, dtype=np.float32).reshape( (32, 32) ) diff --git a/tests/unit/test_cellprofiler_kernel_preparation.py b/tests/unit/test_cellprofiler_kernel_preparation.py index ba9f6173dd..9295d7bcf9 100644 --- a/tests/unit/test_cellprofiler_kernel_preparation.py +++ b/tests/unit/test_cellprofiler_kernel_preparation.py @@ -148,3 +148,83 @@ def record_compile(dispatcher, signature): timeout=120, ) assert result.returncode == 0, result.stdout + result.stderr + + +def test_illumination_preparation_covers_masked_polynomial_and_array_views(tmp_path): + """READY must include every canonical pixel/mask mutability signature.""" + script = textwrap.dedent(""" + from contextlib import ExitStack + from unittest.mock import patch + import numpy as np + from openhcs.core.callable_contract import prepare_processing_callable + from openhcs.processing.backends.cellprofiler import illumination + + dispatchers = ( + illumination._fit_polynomial_unmasked_gram_numba, + illumination._fit_polynomial_unmasked_rhs_numba, + illumination._fit_polynomial_normal_equations_numba, + illumination._evaluate_polynomial_surface_numba, + ) + assert all(not dispatcher.signatures for dispatcher in dispatchers) + prepare_processing_callable(illumination.correct_illumination_calculate) + signatures = tuple(tuple(dispatcher.signatures) for dispatcher in dispatchers) + assert all(signatures), "A reached masked polynomial kernel was not prepared" + + def reject_late_compilation(signature): + raise AssertionError("Polynomial kernel compiled or loaded after READY: " + str(signature)) + + # The polynomial spans the exact solver feature basis, so the expected + # answer is independent of production warmup and implementation details. + shape = (11, 13) + y, x = np.indices(shape, dtype=np.float64) + expected = 1.0 + x / shape[1] + y / shape[0] + (x / shape[1]) ** 2 + mask = np.ones(shape, dtype=bool) + mask[2:4, 3:5] = False + with ExitStack() as stack: + for dispatcher in dispatchers: + stack.enter_context(patch.object(dispatcher, "compile", reject_late_compilation)) + for dtype in (np.float32, np.float64): + for order in ("C", "F", "reversed"): + for readonly_image in (False, True): + image = np.array(expected, dtype=dtype, order="F" if order == "F" else "C") + selected_mask = mask.copy(order="F" if order == "F" else "C") + if order == "reversed": + image = image[::-1, ::-1] + selected_mask = selected_mask[::-1, ::-1] + if readonly_image: + image.flags.writeable = False + for mask_mode in ("absent", "writable", "readonly"): + active_mask = None if mask_mode == "absent" else selected_mask.copy(order="K") + if mask_mode == "readonly": + active_mask.flags.writeable = False + before_image = image.copy() + before_mask = None if active_mask is None else active_mask.copy() + result = illumination.fit_polynomial_surface(image, active_mask) + oracle = expected[::-1, ::-1] if order == "reversed" else expected + np.testing.assert_allclose(result, oracle, atol=1e-6, rtol=1e-6) + np.testing.assert_array_equal(image, before_image) + if active_mask is not None: + np.testing.assert_array_equal(active_mask, before_mask) + assert not np.shares_memory(result, image) + if active_mask is not None: + assert not np.shares_memory(result, active_mask) + with np.testing.assert_raises_regex(ValueError, "mask must match"): + illumination.fit_polynomial_surface(expected, np.ones((3, 4), dtype=bool)) + with np.testing.assert_raises_regex(NotImplementedError, "2-D"): + illumination.fit_polynomial_surface(expected[None, ...], None) + assert tuple(tuple(dispatcher.signatures) for dispatcher in dispatchers) == signatures + """) + environment = os.environ.copy() + environment.update( + {"OPENHCS_CPU_ONLY": "true", "NUMBA_CACHE_DIR": str(tmp_path / "polynomial-kernels")} + ) + result = subprocess.run( + (sys.executable, "-c", script), + cwd=Path(__file__).parents[2], + env=environment, + check=False, + capture_output=True, + text=True, + timeout=90, + ) + assert result.returncode == 0, result.stdout + result.stderr From 66b485034217f5ab86410d67aafa881facb8e138 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 15:40:57 -0400 Subject: [PATCH 080/125] refactor: snapshot dense moments before public domain effects Keep the original Sequence contract and NumPy allocation/error order without exact-type admission probes. The storage parent owns shared center allocation and division; dense storage supplies the snapshot before effectful domain evaluation. Refs #384. Eleven original-parent callback and mutation controls, 29 focused controls and exact saved Wound/Track coordinate ABI replay pass. The original rejected guard implementation and R0 failure remain retained externally. --- openhcs/core/runtime_object_labels.py | 95 ++++++++++++------- .../test_dense_object_coordinate_reduction.py | 82 +++++++++++++++- 2 files changed, 139 insertions(+), 38 deletions(-) diff --git a/openhcs/core/runtime_object_labels.py b/openhcs/core/runtime_object_labels.py index fac841b9bb..a503a78b14 100644 --- a/openhcs/core/runtime_object_labels.py +++ b/openhcs/core/runtime_object_labels.py @@ -6,6 +6,7 @@ from collections.abc import ( Callable, Hashable, + Iterable, MutableMapping, Sequence, ) @@ -1112,14 +1113,30 @@ def axis_centers( if sparse_labels.has_slice_index else (sparse_labels.y_column, sparse_labels.x_column) ) + return self.coordinate_centers( + counts, + ( + np.bincount( + object_ids, + weights=array[:, coordinate_column], + minlength=max_label + 1, + ) + for coordinate_column in coordinate_columns + ), + maximum_label=max_label, + ) + + @staticmethod + def coordinate_centers( + counts: np.ndarray, + coordinate_sums: Iterable[np.ndarray], + *, + maximum_label: int, + ) -> tuple[tuple[np.ndarray, ...], np.ndarray]: + """Allocate and divide each coordinate sum in its declared ID domain.""" axis_centers: list[np.ndarray] = [] - for coordinate_column in coordinate_columns: - sums = np.bincount( - object_ids, - weights=array[:, coordinate_column], - minlength=max_label + 1, - ) - centers = np.full(max_label + 1, np.nan, dtype=np.float64) + for sums in coordinate_sums: + centers = np.full(maximum_label + 1, np.nan, dtype=np.float64) np.divide(sums, counts, out=centers, where=counts > 0) axis_centers.append(centers) return tuple(axis_centers), counts @@ -1205,30 +1222,41 @@ def prepare_coordinates(cls) -> None: labels = np.array([[0, 1]], dtype=np.int32) for writeable in (True, False): labels.flags.writeable = writeable - _dense_label_coordinate_centers_numba(labels.ravel(), 1, 2, 1, 2) + _dense_label_coordinate_moments_numba(labels.ravel(), 1, 2, 1) dense_label_centers_2d_numba(labels, 1) def axis_centers( self, labels: object, *, domain: Sequence[int] ) -> tuple[tuple[np.ndarray, ...], np.ndarray]: + array = np.asarray(labels) if ( - type(labels) is not np.ndarray - or labels.dtype != np.dtype(np.int32) - or labels.ndim not in (2, 3) - or any(size > np.iinfo(np.int32).max for size in labels.shape) - or type(domain) is not tuple - or any(type(value) is not int for value in domain) - or max(domain, default=0) >= np.iinfo(np.intp).max + array.dtype != np.dtype(np.int32) + or array.ndim not in (2, 3) + or any(size > np.iinfo(np.int32).max for size in array.shape) ): return super().axis_centers(labels, domain=domain) - maximum_label = max(int(labels.max(initial=0)), max(domain, default=0)) - centers, counts = _dense_label_coordinate_centers_numba( - labels.ravel(order="C"), - labels.shape[-2], labels.shape[-1], maximum_label, labels.ndim, + coordinate_columns = tuple(range(3 - array.ndim, 3)) + sums, pixel_counts = _dense_label_coordinate_moments_numba( + array.ravel(order="C"), + array.shape[-2], array.shape[-1], int(array.max(initial=0)), ) - return ( - tuple(centers[:, axis].copy() for axis in range(labels.ndim)), - counts.copy(), + # Like sparse conversion, the moments snapshot precedes domain callbacks. + object_ids = np.flatnonzero(pixel_counts) + max_domain_label = max(domain, default=0) + maximum_label = max(int(object_ids.max(initial=0)), max_domain_label) + counts = np.bincount(object_ids, minlength=maximum_label + 1) + counts[object_ids] = pixel_counts[object_ids] + return self.coordinate_centers( + counts, + ( + np.bincount( + object_ids, + weights=sums[object_ids, coordinate_column], + minlength=maximum_label + 1, + ) + for coordinate_column in coordinate_columns + ), + maximum_label=maximum_label, ) def stack_planes( @@ -1593,12 +1621,11 @@ def object_label_axis_centers( @njit(cache=True) -def _dense_label_coordinate_centers_numba( +def _dense_label_coordinate_moments_numba( flat_labels: np.ndarray, height: int, width: int, maximum_label: int, - coordinate_count: int, ) -> tuple[np.ndarray, np.ndarray]: """Reduce dense positive labels once in their declared row-major geometry.""" sums = np.zeros((maximum_label + 1, 3), dtype=np.float64) @@ -1616,14 +1643,7 @@ def _dense_label_coordinate_centers_numba( sums[label_id, 1] += y sums[label_id, 2] += x counts[label_id] += 1 - centers = np.empty((maximum_label + 1, coordinate_count), dtype=np.float64) - for label_id in range(maximum_label + 1): - for axis in range(coordinate_count): - centers[label_id, axis] = ( - np.nan if counts[label_id] == 0 - else sums[label_id, axis + 3 - coordinate_count] / counts[label_id] - ) - return centers, counts + return sums, counts @njit(cache=True) @@ -1632,9 +1652,16 @@ def dense_label_centers_2d_numba( ) -> np.ndarray: """Validate compiled two-dimensional geometry and return its y/x centers.""" height, width = labels.shape - centers, _counts = _dense_label_coordinate_centers_numba( - labels.ravel(), height, width, label_count, 2 + sums, counts = _dense_label_coordinate_moments_numba( + labels.ravel(), height, width, label_count ) + centers = np.empty((label_count + 1, 2), dtype=np.float64) + for label_id in range(label_count + 1): + for axis in range(2): + centers[label_id, axis] = ( + np.nan if counts[label_id] == 0 + else sums[label_id, axis + 1] / counts[label_id] + ) return centers diff --git a/tests/unit/test_dense_object_coordinate_reduction.py b/tests/unit/test_dense_object_coordinate_reduction.py index b62bd89290..f3cc02442e 100644 --- a/tests/unit/test_dense_object_coordinate_reduction.py +++ b/tests/unit/test_dense_object_coordinate_reduction.py @@ -124,26 +124,100 @@ def test_invalid_shape_keeps_original_error(shape): assert str(current.value) == str(original.value) +@pytest.mark.parametrize("failure", (None, "iterate", 1, 2, 3, 4, 5)) +def test_domain_callbacks_follow_pixel_snapshot_and_original_allocation_order(failure): + outcomes = [] + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + labels = np.array([[1, 1], [2, 0]], dtype=np.int32) + effects = [] + + class DomainInteger(int): + def __gt__(self, other): + effects.append("compare") + return super().__gt__(other) + + def __add__(self, other): + effects.append("add") + result = super().__add__(other) + return float(result) if effects.count("add") == failure else result + + class MutatingDomain: + def __iter__(self): + effects.append("iterate") + labels[:] = 3 + labels.shape = (1, 2, 2) + if failure == "iterate": + raise RuntimeError("Domain iteration failed after changing pixels") + yield DomainInteger(4) + + try: + result = call(labels, MutatingDomain()) + except (RuntimeError, TypeError) as error: + outcome = (type(error), str(error)) + else: + outcome = result + outcomes.append((outcome, effects, labels.copy())) + + if failure is None: + assert_centers(outcomes[0][0], outcomes[1][0]) + np.testing.assert_array_equal(outcomes[1][0][1], (0, 2, 1, 0, 0)) + assert len(outcomes[1][0][0]) == 2 + else: + assert outcomes[0][0] == outcomes[1][0] + assert outcomes[0][1] == outcomes[1][1] + np.testing.assert_array_equal(outcomes[0][2], outcomes[1][2]) + + +@pytest.mark.parametrize("domain", ((4.0,), (np.int64(np.iinfo(np.int64).max),), (2**63,))) +def test_domain_minlength_errors_and_warnings_remain_numpy_owned(domain): + outcomes = [] + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + with warnings.catch_warnings(record=True) as observed: + warnings.simplefilter("always") + try: + call(np.array([[1, 0], [2, 1]], dtype=np.int32), domain) + except (TypeError, ValueError, OverflowError) as error: + outcome = (type(error), str(error)) + else: + raise AssertionError("Invalid minlength unexpectedly succeeded") + outcomes.append((outcome, [(type(w.message), str(w.message)) for w in observed])) + assert outcomes[0] == outcomes[1] + + +def test_invalid_geometry_precedes_domain_iteration(): + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + effects = [] + + class Domain: + def __iter__(self): + effects.append("iterate") + yield 1 + + with pytest.raises(ValueError, match="SparseIJVLabelRows.from_dense_stack"): + call(np.zeros((2, 3, 4, 5), dtype=np.int32), Domain()) + assert not effects + + def test_primary_objects_preparation_readies_all_admitted_storage_signatures(tmp_path): script = textwrap.dedent(""" from unittest.mock import patch import numpy as np from openhcs.core.callable_contract import prepare_processing_callable from openhcs.core.runtime_object_labels import ( - _dense_label_coordinate_centers_numba, dense_label_centers_2d_numba, + _dense_label_coordinate_moments_numba, dense_label_centers_2d_numba, object_label_axis_centers, ) from openhcs.processing.backends.cellprofiler.primary_objects import identify_primary_objects - assert not _dense_label_coordinate_centers_numba.signatures + assert not _dense_label_coordinate_moments_numba.signatures prepare_processing_callable(identify_primary_objects) - kernels = (_dense_label_coordinate_centers_numba, dense_label_centers_2d_numba) + kernels = (_dense_label_coordinate_moments_numba, dense_label_centers_2d_numba) signatures = tuple(tuple(kernel.signatures) for kernel in kernels) assert all(signatures) def reject_compilation(signature): raise AssertionError(('Late coordinate compilation', str(signature))) - with patch.object(_dense_label_coordinate_centers_numba, 'compile', + with patch.object(_dense_label_coordinate_moments_numba, 'compile', side_effect=reject_compilation), patch.object( dense_label_centers_2d_numba, 'compile', side_effect=reject_compilation): for shape in ((4, 5), (3, 4, 5)): From eb2f23efdf6c9f7d4da0e5692567b3535f23f257 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 15:47:49 -0400 Subject: [PATCH 081/125] docs: qualify current native frontier and merged readiness repair Refs #384, #419, #433, #435 and #450. Preserve timing scopes and original failed gates. --- .../shared_runtime_plumbing_384_20261001.rst | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index d8a3c6bfb9..16cf3492e0 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1617,3 +1617,128 @@ this priority change. No rejected route is promoted because its preparation is already complete. The tracking renderer is being investigated in an independent main142 worktree with actual 21-frame producer inputs and exact native panel replay; no saved-image repair is included or claimed here yet. + +Library readiness repair and latest qualification, 2026-10-02 +------------------------------------------------------------ + +Main 3f955d3fb (MCP sampling) and cf5a83f89 (fixture CLI repair) are normally +merged. The MCP/prepared-geometry/automatic-role integration gate records 354 +passes and nine failures. All nine fail identically on clean main3f955 with the +same readonly native-binary bindings: eight fixture declarations omit the +required server, and one expected stream-argument mapping omits display_config. +The first clean-main attempt refused a missing native extension at collection; +that original failure is retained before the exact-bound native supplement. +These are existing fixture failures, not a passing full suite. Numerical and +dependency sources are unchanged by the fixture-CLI merge. + +The actual Illumination3 first run reached an unprepared masked polynomial +specialization. Its Numba artifacts were written inside the measured polynomial +step: 0.526s versus 0.016s when already cached. The existing callable preparation +hook now prepares masked/unmasked and readonly canonical image/mask signatures +before READY. A fresh-cache subprocess forbids all later compilation and cache +loads across 36 dtype/layout/mutability/mask combinations and verifies an +independent polynomial oracle, input isolation and geometry errors. Twenty +preparation/fitted-illumination controls pass, with two existing skips. The fix +ships independently in merged PR474, formally closing issue473; original R0 +passes all three roots and original R1 passes its unchanged 160-second budget. +Main a6c18f054 is normally merged back into this branch. Its full receipt is +``docs/validation/polynomial-library-readiness-473-20261002.rst``. + +The clean frozen integrated source54708367b has fresh ordinary CPU5 one-worker, +one-thread observations using the existing cache, default OUTCOMES observer and +default memory policy: + +.. list-table:: Fresh post-repair observations, seconds + :header-rows: 1 + + * - Case + - Compilation + - Execution + - Pipeline total + * - Illumination3 + - 0.962373 + - 0.647611 + - 1.961545 + * - WoundHealing + - 0.346341 + - 3.916717 + - 4.575604 + +Startup/library/kernel preparation and shutdown remain outside pipeline clocks. +These are single observations, with no matched repair speedup or statistical +regression claim. Wound is slower than its retained earlier single observation; +that result is preserved rather than substituting the better sample. Both +authored Illumination3 NPY images and Wound's complete two-row Image.csv agree +byte-for-byte with the qualified unchanged references. Existing complete saved +inventory and CP-tolerance comparison gates pass. Native science is transitive +through those fully qualified references. The first validator incorrectly +expected two Wound tables rather than two rows; its failure remains retained, +and the V2 supplement checks the actual authored inventory without changing any +production comparison policy. Source, dependencies, interpreter, native binaries +and all inputs remain frozen. Receipt: +``/var/tmp/openhcs-pr394-masked-prewarm-ordinary-science-v2-20261002.json`` +(SHA256 21231dffa14dbe17bf824a4258f677e7d656122036da81e0b3d8c9c8bc2f9c02). + +Additional native qualification and remaining execution frontier +--------------------------------------------------------------- + +Current9aa Vitra uses one selected joined source set containing all four physical +resources, rather than two channel-derived image sets. Its two warm native +repetitions average 3.044868s; OpenHCS is 1.615775s execution and 2.908514s total. +Complete saved science passes for 2,233 table rows and its authored image in both +native repetitions. Earlier source-set controller refusals remain retained. +Receipt: +``/var/tmp/openhcs-current-vitra-native-timing-science-v3-20261002.json``. + +Illumination3's two warm native repetitions average 0.603076s. Its authored +pipeline saves two NPY images and no CSV. Complete existing inventory and image +science pass in both repetitions. The retained first OpenHCS observation is +1.309110s execution and 2.603767s total, containing the late polynomial +specialization described above; the fresh post-repair observation is recorded +separately. Cache-state differences prevent treating these as a matched repair +speedup. The global weakest full-catalog case remains unknown. Receipt: +``/var/tmp/openhcs-current-illumination3-native-timing-science-v1-20261002.json``. + +Wound's actual worker profile partitions 3.387111s into 1.536850s of disjoint raw +processing and 1.850261s of other runtime/recording work. The 22 stack operations +include 0.4036s in 16 aggregate image stacks, 0.1481s in object variants, 0.0538s +in main-flow/cache and 0.0023s in initial loading. Ten source-record +normalizations account for 0.2436s of the 0.5510s normalization envelope. These +are profiled costs and overlap the earlier source-query attribution; none are +accepted saved seconds. + +Actual Wound ON_DEMAND QA images are not forced by terminal OUTCOMES publication. +However, raw QA planes borrow object-variant pixels and shared validity masks; +their current aggregates establish independent mutable artifact buffers. +Deferral with equivalent eager per-plane copies cannot remove those copy bytes, +and zero-copy adoption without an ownership/lifetime proof is unsound. Existing +ArrayBridge geometry validation also forces concrete arrays at payload +construction. No lazy wrapper or second mutable identity cache is promoted. +The global AST owner/lifetime audit retains a bounded actual alias-capture plan: +``/var/tmp/openhcs-current9aa-wound-copy-lifetime-reassessment-20261002.json``. + +An external shared-geometry convex prototype preserves both actual saved input +graphs exactly. The outer 256-level call's warm local median changes from +0.260906s to 0.023542s, saving 0.237364s; the nested 16-level call overlaps it +and is never added as independent work. This is useful local evidence, not a +production pipeline gain. The nominal owner audit identifies existing shared +column-envelope and Bresenham authorities and a disconnected 15-kernel legacy +component; production migration and end-to-end qualification remain separate. +The shared dense-coordinate prototype also has exact actual Wound/Track input +replays and passing scoped R0/R1 after removing duplicated admission logic. +Its pre-domain allocation behavior for very sparse large label IDs remains +under review. Neither prototype is integrated here or counted as a speedup. + +Independent tracking PR468 is synced to main and passes 19 focused tests plus +unchanged scoped R0/R1. It remains draft: native saved-image differences shrink +from 77,726 to 3,340 text pixels, but full native PNG parity is still RED. It +formally links issue467. No parity gate, tolerance, scientific output or renderer +environment is weakened, and no tracking repair is included in this branch. + +Remaining acceptance still includes installed-root issue419 IPO/Shape and +Morph/DISTANCE consumers; issue433's original 13.36GB OOM allocation owner and +repeated READY behavior; installed-root issue435; and issue450's installed +prepared-Resize consumer. The original whole-branch scalar-classifier R0 RED +remains separately retained. These draft closing references do not establish +completion. Full native scaling, all30 benchmark reruns and fresh figures remain +outstanding, and the long-running performance goal stays active. From b4700288f6762674e5d79f95730e20cf6d348f16 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 15:57:03 -0400 Subject: [PATCH 082/125] perf: share exact CP geometry in incremental quantized hulls Reuse the label column-envelope walk and Bresenham segment rasterizer across illumination, label hulls, and worms. Keep original float quantization and recursive level semantics, preserve owned public line arrays, and prepare every reached geometry signature before execution. Remove the disconnected legacy diamond-hull component. Refs #394. --- .../backends/cellprofiler/illumination.py | 718 +++--------------- .../backends/cellprofiler/label_geometry.py | 90 ++- .../backends/cellprofiler/worm_geometry.py | 78 +- .../test_cellprofiler_kernel_preparation.py | 78 ++ tests/unit/test_cellprofiler_worm_geometry.py | 42 + 5 files changed, 324 insertions(+), 682 deletions(-) diff --git a/openhcs/processing/backends/cellprofiler/illumination.py b/openhcs/processing/backends/cellprofiler/illumination.py index 1dab00b4e5..3c53edf1f8 100644 --- a/openhcs/processing/backends/cellprofiler/illumination.py +++ b/openhcs/processing/backends/cellprofiler/illumination.py @@ -71,6 +71,7 @@ from openhcs.core.runtime_profile import RuntimeProfiler from openhcs.processing.backends.cellprofiler.label_geometry import ( CellProfilerLabelHull, + _cellprofiler_column_envelope_vertices_numba, ) from openhcs.processing.backends.cellprofiler.morphology import ( MorphologyBackendStrategy, @@ -84,6 +85,7 @@ ) from openhcs.processing.backends.cellprofiler.worm_geometry import ( _cellprofiler_line_points, + _fill_cellprofiler_line_points_numba, ) from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract @@ -1804,6 +1806,12 @@ def prepare_backend(self) -> None: self.smooth_background_plane( image, mask=mask, filter_size=3, morphology=morphology ) + _cellprofiler_convex_hull_transform(np.asfortranarray(image), mask) + endpoints = np.array([0, 3], dtype=int) + _cellprofiler_line_points(endpoints, endpoints, endpoints[::-1], endpoints) + CellProfilerLabelHull.from_labels( + (image > 0.5).astype(np.int32), np.array([1], dtype=np.int32) + ).vertices() def smooth_background_plane( self, @@ -1906,44 +1914,101 @@ def _cellprofiler_convex_hull_transform( ) scaled = np.maximum(scaled, rough) scaled = scaled.astype(np.int32) - unique_levels = np.unique(scaled) - output_levels = np.full( - image_array.shape, - int(unique_levels[0]), - dtype=np.int32, - ) + output_levels = _incremental_quantized_hulls_numba(scaled) + return scale[output_levels] - for level in unique_levels[1:]: - level_mask = scaled >= int(level) - label_hull = CellProfilerLabelHull.from_labels( - level_mask.astype(np.int32), - np.array([1], dtype=np.int32), - ) - hull, counts = label_hull.vertices() - if counts[0] == 0: + +@njit(cache=True) +def _paint_column_envelope_level_numba( + vertices: np.ndarray, + vertex_count: int, + output: np.ndarray, + assigned: np.ndarray, + level: int, + line_rows: np.ndarray, + line_columns: np.ndarray, +) -> None: + """Fill each column enclosed by the canonical CP rasterized hull edges.""" + width = output.shape[1] + low = np.full(width, np.iinfo(np.int64).max, np.int64) + high = np.full(width, np.iinfo(np.int64).min, np.int64) + for segment in range(vertex_count): + following = (segment + 1) % vertex_count + count = _fill_cellprofiler_line_points_numba( + vertices[segment, 0], + vertices[segment, 1], + vertices[following, 0], + vertices[following, 1], + line_rows, + line_columns, + 0, + ) + for position in range(count): + column = line_columns[position] + row = line_rows[position] + low[column] = min(low[column], row) + high[column] = max(high[column], row) + for column in range(width): + if low[column] == np.iinfo(np.int64).max: continue - vertices = hull[:, 1:] - next_vertices = np.roll(vertices, -1, axis=0) - _indexes, _counts, rows, columns = _cellprofiler_line_points( - vertices[:, 0], - vertices[:, 1], - next_vertices[:, 0], - next_vertices[:, 1], - ) - row_minimum = np.full( - image_array.shape[1], np.iinfo(np.int64).max, dtype=np.int64 - ) - row_maximum = np.full( - image_array.shape[1], np.iinfo(np.int64).min, dtype=np.int64 - ) - np.minimum.at(row_minimum, columns, rows) - np.maximum.at(row_maximum, columns, rows) - for column in np.unique(columns): - output_levels[ - int(row_minimum[column]) : int(row_maximum[column]) + 1, - int(column), - ] = int(level) - return scale[output_levels] + for row in range(low[column], high[column] + 1): + if not assigned[row, column]: + output[row, column] = level + assigned[row, column] = True + + +@njit(cache=True) +def _incremental_quantized_hulls_numba(scaled: np.ndarray) -> np.ndarray: + """Reuse cumulative column extrema for descending quantized level sets.""" + height, width = scaled.shape + flat = scaled.ravel() + levels = np.unique(flat) + # Index the observed codes, rather than allocating across a possibly sparse + # or invalid int32 range produced by the original floating-point cast. + counts = np.zeros(levels.size, np.int64) + for value in flat: + counts[np.searchsorted(levels, value)] += 1 + offsets = np.zeros(levels.size + 1, np.int64) + for index in range(levels.size): + offsets[index + 1] = offsets[index] + counts[index] + cursors = offsets.copy() + positions = np.empty(flat.size, np.int64) + for position in range(flat.size): + index = np.searchsorted(levels, flat[position]) + positions[cursors[index]] = position + cursors[index] += 1 + row_minimum = np.full(width, np.iinfo(np.int64).max, np.int64) + row_maximum = np.full(width, np.iinfo(np.int64).min, np.int64) + vertices = np.empty((width * 2, 2), np.int64) + line_rows = np.empty(max(height, width), np.int64) + line_columns = np.empty(line_rows.size, np.int64) + output = np.full((height, width), levels[0], np.int32) + assigned = np.zeros((height, width), np.bool_) + for level_index in range(levels.size - 1, 0, -1): + changed = False + for cursor in range(offsets[level_index], offsets[level_index + 1]): + position = positions[cursor] + row, column = position // width, position % width + if row < row_minimum[column]: + row_minimum[column] = row + changed = True + if row > row_maximum[column]: + row_maximum[column] = row + changed = True + if changed: + vertex_count = _cellprofiler_column_envelope_vertices_numba( + row_minimum, row_maximum, 0, vertices + ) + _paint_column_envelope_level_numba( + vertices, + vertex_count, + output, + assigned, + levels[level_index], + line_rows, + line_columns, + ) + return output @dataclass(frozen=True, slots=True) @@ -2001,587 +2066,6 @@ def _restore_masked_pixels( return result -@njit(cache=True) -def _exact_level_set_convex_hull_smoothing_numba( - image: np.ndarray, valid_mask: np.ndarray, thresholds: np.ndarray -) -> np.ndarray: - height, width = image.shape - minimum = np.float32(0.0) - maximum = np.float32(0.0) - found_valid = False - valid_pixel_count = 0 - for y in range(height): - for x in range(width): - if not valid_mask[y, x]: - continue - valid_pixel_count += 1 - value = image[y, x] - if not found_valid: - minimum = value - maximum = value - found_valid = True - else: - if value < minimum: - minimum = value - if value > maximum: - maximum = value - output = np.empty((height, width), dtype=np.float32) - for y in range(height): - for x in range(width): - output[y, x] = minimum if valid_mask[y, x] else np.float32(0.0) - if not found_valid or maximum <= minimum: - return output - row_count2 = height * 2 + 1 - min_col_by_row = np.empty(row_count2, dtype=np.int64) - max_col_by_row = np.empty(row_count2, dtype=np.int64) - point_capacity = max(2, row_count2 * 2) - point_x = np.empty(point_capacity, dtype=np.int64) - point_y = np.empty(point_capacity, dtype=np.int64) - hull_x = np.empty(point_capacity * 2, dtype=np.int64) - hull_y = np.empty(point_capacity * 2, dtype=np.int64) - bucket_counts = np.zeros(thresholds.size, dtype=np.int64) - active_pixel_count = _count_convex_hull_threshold_buckets( - image, valid_mask, thresholds, bucket_counts - ) - if active_pixel_count == 0: - return output - bucket_offsets = np.empty(thresholds.size + 1, dtype=np.int64) - offset = 0 - for bucket_index in range(thresholds.size): - bucket_offsets[bucket_index] = offset - offset += bucket_counts[bucket_index] - bucket_counts[bucket_index] = 0 - bucket_offsets[thresholds.size] = offset - bucket_rows = np.empty(active_pixel_count, dtype=np.int64) - bucket_cols = np.empty(active_pixel_count, dtype=np.int64) - _fill_convex_hull_threshold_buckets( - image, - valid_mask, - thresholds, - bucket_offsets, - bucket_counts, - bucket_rows, - bucket_cols, - ) - assigned = np.zeros((height, width), dtype=np.bool_) - for row_index in range(row_count2): - min_col_by_row[row_index] = 9223372036854775807 - max_col_by_row[row_index] = -9223372036854775807 - assigned_count = 0 - for level_index in range(thresholds.size - 1, -1, -1): - start = bucket_offsets[level_index] - end = bucket_offsets[level_index] + bucket_counts[level_index] - if start == end: - continue - changed_extrema = False - for bucket_position in range(start, end): - y = bucket_rows[bucket_position] - x = bucket_cols[bucket_position] - if _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y - 1, 2 * x): - changed_extrema = True - if _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y + 1, 2 * x): - changed_extrema = True - if _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y, 2 * x - 1): - changed_extrema = True - if _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y, 2 * x + 1): - changed_extrema = True - if not changed_extrema: - continue - point_count = _emit_diamond_extreme_points( - min_col_by_row, max_col_by_row, point_x, point_y - ) - if point_count == 0: - continue - hull_count = _monotone_chain_hull(point_x, point_y, point_count, hull_x, hull_y) - assigned_count += _paint_convex_hull( - output, - assigned, - False, - valid_mask, - thresholds[level_index], - hull_x, - hull_y, - hull_count, - ) - if assigned_count >= valid_pixel_count: - break - return output - - -@njit(cache=True) -def _count_convex_hull_threshold_buckets( - image: np.ndarray, - valid_mask: np.ndarray, - thresholds: np.ndarray, - bucket_counts: np.ndarray, -) -> int: - height, width = image.shape - active_pixel_count = 0 - for y in range(height): - for x in range(width): - if not valid_mask[y, x]: - continue - bucket_index = _last_threshold_index_not_greater_than( - thresholds, image[y, x] - ) - if bucket_index < 0: - continue - bucket_counts[bucket_index] += 1 - active_pixel_count += 1 - return active_pixel_count - - -@njit(cache=True) -def _fill_convex_hull_threshold_buckets( - image: np.ndarray, - valid_mask: np.ndarray, - thresholds: np.ndarray, - bucket_offsets: np.ndarray, - bucket_counts: np.ndarray, - bucket_rows: np.ndarray, - bucket_cols: np.ndarray, -) -> None: - height, width = image.shape - for y in range(height): - for x in range(width): - if not valid_mask[y, x]: - continue - bucket_index = _last_threshold_index_not_greater_than( - thresholds, image[y, x] - ) - if bucket_index < 0: - continue - bucket_position = bucket_offsets[bucket_index] + bucket_counts[bucket_index] - bucket_rows[bucket_position] = y - bucket_cols[bucket_position] = x - bucket_counts[bucket_index] += 1 - - -@njit(cache=True) -def _last_threshold_index_not_greater_than( - thresholds: np.ndarray, value: np.float32 -) -> int: - low = 0 - high = thresholds.size - while low < high: - middle = (low + high) // 2 - if thresholds[middle] <= value: - low = middle + 1 - else: - high = middle - return low - 1 - - -@njit(cache=True) -def _emit_diamond_extreme_points( - min_col_by_row: np.ndarray, - max_col_by_row: np.ndarray, - point_x: np.ndarray, - point_y: np.ndarray, -) -> int: - point_count = 0 - for row_index in range(max_col_by_row.size): - max_col = max_col_by_row[row_index] - if max_col < -9223372036854775800: - continue - row2 = row_index - 1 - min_col = min_col_by_row[row_index] - point_x[point_count] = row2 - point_y[point_count] = min_col - point_count += 1 - if max_col != min_col: - point_x[point_count] = row2 - point_y[point_count] = max_col - point_count += 1 - return point_count - - -@njit(cache=True) -def _collect_diamond_extreme_points( - image: np.ndarray, - valid_mask: np.ndarray, - threshold: np.float32, - min_col_by_row: np.ndarray, - max_col_by_row: np.ndarray, - point_x: np.ndarray, - point_y: np.ndarray, -) -> int: - height, width = image.shape - row_count2 = height * 2 + 1 - for row_index in range(row_count2): - min_col_by_row[row_index] = 9223372036854775807 - max_col_by_row[row_index] = -9223372036854775807 - for y in range(height): - for x in range(width): - if valid_mask[y, x] and image[y, x] >= threshold: - _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y - 1, 2 * x) - _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y + 1, 2 * x) - _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y, 2 * x - 1) - _add_diamond_vertex(min_col_by_row, max_col_by_row, 2 * y, 2 * x + 1) - return _emit_diamond_extreme_points( - min_col_by_row, max_col_by_row, point_x, point_y - ) - - -@njit(cache=True) -def _exact_level_set_convex_hull_smoothing_reference_numba( - image: np.ndarray, valid_mask: np.ndarray, thresholds: np.ndarray -) -> np.ndarray: - height, width = image.shape - minimum = np.float32(0.0) - maximum = np.float32(0.0) - found_valid = False - for y in range(height): - for x in range(width): - if not valid_mask[y, x]: - continue - value = image[y, x] - if not found_valid: - minimum = value - maximum = value - found_valid = True - else: - if value < minimum: - minimum = value - if value > maximum: - maximum = value - output = np.empty((height, width), dtype=np.float32) - for y in range(height): - for x in range(width): - output[y, x] = minimum if valid_mask[y, x] else np.float32(0.0) - if not found_valid or maximum <= minimum: - return output - row_count2 = height * 2 + 1 - min_col_by_row = np.empty(row_count2, dtype=np.int64) - max_col_by_row = np.empty(row_count2, dtype=np.int64) - point_capacity = max(2, row_count2 * 2) - point_x = np.empty(point_capacity, dtype=np.int64) - point_y = np.empty(point_capacity, dtype=np.int64) - hull_x = np.empty(point_capacity * 2, dtype=np.int64) - hull_y = np.empty(point_capacity * 2, dtype=np.int64) - assigned = np.ones((height, width), dtype=np.bool_) - for level_index in range(thresholds.size): - threshold = thresholds[level_index] - point_count = _collect_diamond_extreme_points( - image, - valid_mask, - threshold, - min_col_by_row, - max_col_by_row, - point_x, - point_y, - ) - if point_count == 0: - continue - hull_count = _monotone_chain_hull(point_x, point_y, point_count, hull_x, hull_y) - _paint_convex_hull( - output, assigned, True, valid_mask, threshold, hull_x, hull_y, hull_count - ) - return output - - -@njit(cache=True) -def _add_diamond_vertex( - min_col_by_row: np.ndarray, max_col_by_row: np.ndarray, row2: int, col2: int -) -> bool: - row_index = row2 + 1 - changed = False - if col2 < min_col_by_row[row_index]: - min_col_by_row[row_index] = col2 - changed = True - if col2 > max_col_by_row[row_index]: - max_col_by_row[row_index] = col2 - changed = True - return changed - - -@njit(cache=True) -def _cross_points(ax: int, ay: int, bx: int, by: int, cx: int, cy: int) -> int: - return (bx - ax) * (cy - ay) - (by - ay) * (cx - ax) - - -@njit(cache=True) -def _monotone_chain_hull( - point_x: np.ndarray, - point_y: np.ndarray, - point_count: int, - hull_x: np.ndarray, - hull_y: np.ndarray, -) -> int: - if point_count <= 1: - if point_count == 1: - hull_x[0] = point_x[0] - hull_y[0] = point_y[0] - return point_count - hull_count = 0 - for index in range(point_count): - px = point_x[index] - py = point_y[index] - while ( - hull_count >= 2 - and _cross_points( - hull_x[hull_count - 2], - hull_y[hull_count - 2], - hull_x[hull_count - 1], - hull_y[hull_count - 1], - px, - py, - ) - <= 0 - ): - hull_count -= 1 - hull_x[hull_count] = px - hull_y[hull_count] = py - hull_count += 1 - lower_count = hull_count - for index in range(point_count - 2, -1, -1): - px = point_x[index] - py = point_y[index] - while ( - hull_count > lower_count - and _cross_points( - hull_x[hull_count - 2], - hull_y[hull_count - 2], - hull_x[hull_count - 1], - hull_y[hull_count - 1], - px, - py, - ) - <= 0 - ): - hull_count -= 1 - hull_x[hull_count] = px - hull_y[hull_count] = py - hull_count += 1 - if hull_count > 1: - hull_count -= 1 - return hull_count - - -@njit(cache=True) -def _paint_convex_hull( - output: np.ndarray, - assigned: np.ndarray, - overwrite_assigned: bool, - valid_mask: np.ndarray, - threshold: np.float32, - hull_x: np.ndarray, - hull_y: np.ndarray, - hull_count: int, -) -> int: - if hull_count <= 0: - return 0 - if hull_count == 1: - if hull_x[0] % 2 != 0 or hull_y[0] % 2 != 0: - return 0 - y = hull_x[0] // 2 - x = hull_y[0] // 2 - if ( - y >= 0 - and y < valid_mask.shape[0] - and (x >= 0) - and (x < valid_mask.shape[1]) - and valid_mask[y, x] - and (overwrite_assigned or not assigned[y, x]) - ): - output[y, x] = threshold - was_unassigned = not assigned[y, x] - assigned[y, x] = True - return 1 if was_unassigned else 0 - return 0 - min_row2 = hull_x[0] - max_row2 = hull_x[0] - min_col2 = hull_y[0] - max_col2 = hull_y[0] - for index in range(1, hull_count): - row2 = hull_x[index] - col2 = hull_y[index] - if row2 < min_row2: - min_row2 = row2 - if row2 > max_row2: - max_row2 = row2 - if col2 < min_col2: - min_col2 = col2 - if col2 > max_col2: - max_col2 = col2 - if hull_count == 2: - return _paint_line_hull( - output, - assigned, - overwrite_assigned, - valid_mask, - threshold, - hull_x[0], - hull_y[0], - hull_x[1], - hull_y[1], - min_row2, - max_row2, - min_col2, - max_col2, - ) - image_height, image_width = output.shape - min_y = max(0, _ceil_div2(min_row2)) - max_y = min(image_height - 1, _floor_div2(max_row2)) - min_x = max(0, _ceil_div2(min_col2)) - max_x = min(image_width - 1, _floor_div2(max_col2)) - return _paint_polygon_hull_scanlines( - output, - assigned, - overwrite_assigned, - valid_mask, - threshold, - hull_x, - hull_y, - hull_count, - min_y, - max_y, - min_x, - max_x, - ) - - -@njit(cache=True) -def _paint_polygon_hull_scanlines( - output: np.ndarray, - assigned: np.ndarray, - overwrite_assigned: bool, - valid_mask: np.ndarray, - threshold: np.float32, - hull_x: np.ndarray, - hull_y: np.ndarray, - hull_count: int, - min_y: int, - max_y: int, - min_x: int, - max_x: int, -) -> int: - assigned_delta = 0 - for y in range(min_y, max_y + 1): - query_row2 = y * 2 - left_col2 = 0.0 - right_col2 = 0.0 - found_intersection = False - for index in range(hull_count): - next_index = index + 1 - if next_index == hull_count: - next_index = 0 - row0 = hull_x[index] - col0 = hull_y[index] - row1 = hull_x[next_index] - col1 = hull_y[next_index] - if row0 == row1: - if query_row2 != row0: - continue - edge_left = float(min(col0, col1)) - edge_right = float(max(col0, col1)) - if not found_intersection: - left_col2 = edge_left - right_col2 = edge_right - found_intersection = True - else: - if edge_left < left_col2: - left_col2 = edge_left - if edge_right > right_col2: - right_col2 = edge_right - continue - row_min = min(row0, row1) - row_max = max(row0, row1) - if query_row2 < row_min or query_row2 > row_max: - continue - row_fraction = (query_row2 - row0) / (row1 - row0) - intersection_col2 = col0 + row_fraction * (col1 - col0) - if not found_intersection: - left_col2 = intersection_col2 - right_col2 = intersection_col2 - found_intersection = True - else: - if intersection_col2 < left_col2: - left_col2 = intersection_col2 - if intersection_col2 > right_col2: - right_col2 = intersection_col2 - if not found_intersection: - continue - scan_min_x = max(min_x, int(np.ceil(left_col2 / 2.0 - 1e-09))) - scan_max_x = min(max_x, int(np.floor(right_col2 / 2.0 + 1e-09))) - for x in range(scan_min_x, scan_max_x + 1): - if not valid_mask[y, x]: - continue - if not overwrite_assigned and assigned[y, x]: - continue - output[y, x] = threshold - if not assigned[y, x]: - assigned_delta += 1 - assigned[y, x] = True - return assigned_delta - - -@njit(cache=True) -def _ceil_div2(value: int) -> int: - if value >= 0: - return (value + 1) // 2 - return value // 2 - - -@njit(cache=True) -def _floor_div2(value: int) -> int: - if value >= 0: - return value // 2 - return -((-value + 1) // 2) - - -@njit(cache=True) -def _paint_line_hull( - output: np.ndarray, - assigned: np.ndarray, - overwrite_assigned: bool, - valid_mask: np.ndarray, - threshold: np.float32, - x0: int, - y0: int, - x1: int, - y1: int, - min_row2: int, - max_row2: int, - min_col2: int, - max_col2: int, -) -> int: - dx = x1 - x0 - dy = y1 - y0 - length2 = dx * dx + dy * dy - if length2 == 0: - if valid_mask[y0, x0] and (overwrite_assigned or not assigned[y0, x0]): - output[y0, x0] = threshold - was_unassigned = not assigned[y0, x0] - assigned[y0, x0] = True - return 1 if was_unassigned else 0 - return 0 - image_height, image_width = output.shape - min_y = max(0, _ceil_div2(min_row2)) - max_y = min(image_height - 1, _floor_div2(max_row2)) - min_x = max(0, _ceil_div2(min_col2)) - max_x = min(image_width - 1, _floor_div2(max_col2)) - assigned_delta = 0 - for y in range(min_y, max_y + 1): - query_row2 = y * 2 - for x in range(min_x, max_x + 1): - if not valid_mask[y, x]: - continue - if not overwrite_assigned and assigned[y, x]: - continue - query_col2 = x * 2 - dot = (query_row2 - x0) * dx + (query_col2 - y0) * dy - if dot < 0 or dot > length2: - continue - cross = dx * (query_col2 - y0) - dy * (query_row2 - x0) - if cross == 0: - output[y, x] = threshold - if not assigned[y, x]: - assigned_delta += 1 - assigned[y, x] = True - return assigned_delta - - @njit(cache=True) def _rank_median_global_minimum_is_majority_everywhere_numba( image: np.ndarray, diff --git a/openhcs/processing/backends/cellprofiler/label_geometry.py b/openhcs/processing/backends/cellprofiler/label_geometry.py index 61fef5884c..9dc88da3af 100644 --- a/openhcs/processing/backends/cellprofiler/label_geometry.py +++ b/openhcs/processing/backends/cellprofiler/label_geometry.py @@ -356,6 +356,53 @@ def _outline_points_at_positions_numba( return counts, offsets, ys, xs +@njit(cache=True) +def _cellprofiler_column_envelope_vertices_numba( + row_minimum: np.ndarray, + row_maximum: np.ndarray, + column_offset: int, + vertices: np.ndarray, +) -> int: + """Fill caller-owned vertices using CP's exact ordered envelope walk.""" + columns = np.flatnonzero(row_minimum != np.iinfo(np.int64).max) + vertex_count = 0 + for k in range(2 * columns.size): + if k < columns.size: + col = columns[k] + y = row_minimum[col] + else: + col = columns[2 * columns.size - 1 - k] + y = row_maximum[col] + x = col + column_offset + if ( + vertex_count + and vertices[vertex_count - 1, 0] == y + and vertices[vertex_count - 1, 1] == x + ): + continue + while vertex_count >= 2: + previous_y = vertices[vertex_count - 2, 0] + previous_x = vertices[vertex_count - 2, 1] + middle_y = vertices[vertex_count - 1, 0] + middle_x = vertices[vertex_count - 1, 1] + cross = (middle_x - previous_x) * (y - middle_y) - (x - middle_x) * ( + middle_y - previous_y + ) + if cross > 0 or (cross == 0 and middle_x > previous_x and middle_x > x): + break + vertex_count -= 1 + vertices[vertex_count, 0] = y + vertices[vertex_count, 1] = x + vertex_count += 1 + if ( + vertex_count > 1 + and vertices[vertex_count - 1, 0] == vertices[0, 0] + and vertices[vertex_count - 1, 1] == vertices[0, 1] + ): + vertex_count -= 1 + return vertex_count + + @njit(cache=True) def _cellprofiler_hull_vertices_numba( indexes: np.ndarray, @@ -385,43 +432,11 @@ def _cellprofiler_hull_vertices_numba( y = point_y[p] row_minimum[col] = min(row_minimum[col], y) row_maximum[col] = max(row_maximum[col], y) - columns = np.flatnonzero(row_minimum != np.iinfo(np.int64).max) - vertices = np.empty((2 * columns.size, 2), np.int64) - vertex_count = 0 - for k in range(2 * columns.size): - if k < columns.size: - col = columns[k] - y = row_minimum[col] - else: - col = columns[2 * columns.size - 1 - k] - y = row_maximum[col] - x = col + first - if ( - vertex_count - and vertices[vertex_count - 1, 0] == y - and vertices[vertex_count - 1, 1] == x - ): - continue - while vertex_count >= 2: - previous_y = vertices[vertex_count - 2, 0] - previous_x = vertices[vertex_count - 2, 1] - middle_y = vertices[vertex_count - 1, 0] - middle_x = vertices[vertex_count - 1, 1] - cross = (middle_x - previous_x) * (y - middle_y) - (x - middle_x) * ( - middle_y - previous_y - ) - if cross > 0 or (cross == 0 and middle_x > previous_x and middle_x > x): - break - vertex_count -= 1 - vertices[vertex_count, 0] = y - vertices[vertex_count, 1] = x - vertex_count += 1 - if ( - vertex_count > 1 - and vertices[vertex_count - 1, 0] == vertices[0, 0] - and vertices[vertex_count - 1, 1] == vertices[0, 1] - ): - vertex_count -= 1 + column_count = np.count_nonzero(row_minimum != np.iinfo(np.int64).max) + vertices = np.empty((2 * column_count, 2), np.int64) + vertex_count = _cellprofiler_column_envelope_vertices_numba( + row_minimum, row_maximum, first, vertices + ) hull_counts[object_index] = vertex_count for p in range(vertex_count): output[output_count, 0] = indexes[object_index] @@ -621,6 +636,7 @@ def _numpy124_partition_indices_numba( current_depth = stack_depth[stack_size] return indices + def _numpy124_aquicksort_indices(values: np.ndarray) -> np.ndarray: return _numpy124_partition_indices_numba(np.asarray(values)) diff --git a/openhcs/processing/backends/cellprofiler/worm_geometry.py b/openhcs/processing/backends/cellprofiler/worm_geometry.py index c9af20347d..54a8d6102d 100644 --- a/openhcs/processing/backends/cellprofiler/worm_geometry.py +++ b/openhcs/processing/backends/cellprofiler/worm_geometry.py @@ -371,6 +371,47 @@ def sample_control_points( return np.vstack((path_coords[:1, :], sampled, path_coords[-1:, :])) +@njit(cache=True) +def _fill_cellprofiler_line_points_numba( + row: int, + column: int, + end_row: int, + end_column: int, + rows: np.ndarray, + columns: np.ndarray, + output_index: int, +) -> int: + """Fill one inclusive segment with CP's Bresenham endpoint and tie rules.""" + row_delta = abs(row - end_row) + column_delta = abs(column - end_column) + count = max(row_delta, column_delta) + 1 + rows[output_index] = row + columns[output_index] = column + step_row = 1 if end_row > row else -1 + step_column = 1 if end_column > column else -1 + if row_delta >= column_delta: + remainder = 2 * column_delta - row_delta + for offset in range(1, count): + if remainder >= 0: + column += step_column + remainder -= 2 * row_delta + row += step_row + remainder += 2 * column_delta + rows[output_index + offset] = row + columns[output_index + offset] = column + else: + remainder = 2 * row_delta - column_delta + for offset in range(1, count): + if remainder >= 0: + row += step_row + remainder -= 2 * column_delta + column += step_column + remainder += 2 * row_delta + rows[output_index + offset] = row + columns[output_index + offset] = column + return count + + def _cellprofiler_line_points( start_rows: np.ndarray, start_columns: np.ndarray, @@ -393,34 +434,15 @@ def _cellprofiler_line_points( columns = np.empty(rows.size, dtype=int) for segment in range(len(count)): - output_index = int(index[segment]) - row = int(row0[segment]) - column = int(column0[segment]) - rows[output_index] = row - columns[output_index] = column - step_row = 1 if row1[segment] > row0[segment] else -1 - step_column = 1 if column1[segment] > column0[segment] else -1 - - if row_delta[segment] >= column_delta[segment]: - remainder = 2 * int(column_delta[segment]) - int(row_delta[segment]) - for offset in range(1, int(count[segment])): - if remainder >= 0: - column += step_column - remainder -= 2 * int(row_delta[segment]) - row += step_row - remainder += 2 * int(column_delta[segment]) - rows[output_index + offset] = row - columns[output_index + offset] = column - else: - remainder = 2 * int(row_delta[segment]) - int(column_delta[segment]) - for offset in range(1, int(count[segment])): - if remainder >= 0: - row += step_row - remainder -= 2 * int(column_delta[segment]) - column += step_column - remainder += 2 * int(row_delta[segment]) - rows[output_index + offset] = row - columns[output_index + offset] = column + _fill_cellprofiler_line_points_numba( + int(row0[segment]), + int(column0[segment]), + int(row1[segment]), + int(column1[segment]), + rows, + columns, + int(index[segment]), + ) return index, count, rows, columns diff --git a/tests/unit/test_cellprofiler_kernel_preparation.py b/tests/unit/test_cellprofiler_kernel_preparation.py index 9295d7bcf9..c438e686fb 100644 --- a/tests/unit/test_cellprofiler_kernel_preparation.py +++ b/tests/unit/test_cellprofiler_kernel_preparation.py @@ -228,3 +228,81 @@ def reject_late_compilation(signature): timeout=90, ) assert result.returncode == 0, result.stdout + result.stderr + + +def test_illumination_preparation_covers_shared_hull_and_line_geometry(tmp_path): + """Prepared geometry must not compile or load kernels during execution.""" + script = textwrap.dedent(""" + from contextlib import ExitStack + from unittest.mock import patch + import numpy as np + import centrosome.filter + import centrosome.cpmorphology + from openhcs.core.callable_contract import prepare_processing_callable + from openhcs.processing.backends.cellprofiler import illumination, label_geometry, worm_geometry + + dispatchers = ( + illumination._incremental_quantized_hulls_numba, + illumination._paint_column_envelope_level_numba, + label_geometry._cellprofiler_column_envelope_vertices_numba, + worm_geometry._fill_cellprofiler_line_points_numba, + ) + assert all(not dispatcher.signatures for dispatcher in dispatchers) + prepare_processing_callable(illumination.correct_illumination_calculate) + before = tuple(tuple(dispatcher.signatures) for dispatcher in dispatchers) + assert all(before) + layouts = {signature[0].layout for signature in dispatchers[0].signatures} + assert layouts == {"C", "F"}, layouts + + def reject_late_compilation(signature): + raise AssertionError("Shared geometry compiled or loaded after READY: " + str(signature)) + + rng = np.random.default_rng(419) + source = rng.random((13, 17)) + partial_mask = rng.random(source.shape) > 0.2 + with ExitStack() as stack: + for dispatcher in dispatchers: + stack.enter_context(patch.object(dispatcher, "compile", reject_late_compilation)) + for dtype in (np.float32, np.float64): + for view in (source.astype(dtype), np.asfortranarray(source, dtype=dtype), source[::-1, ::-1].astype(dtype)): + for writeable in (True, False): + view.flags.writeable = writeable + pixels = view.copy() + for mask in (None, partial_mask): + mask_before = None if mask is None else mask.copy() + for levels in (16, 256): + expected = centrosome.filter.convex_hull_transform(view, mask=mask, levels=levels) + actual = illumination._cellprofiler_convex_hull_transform(view, mask, levels=levels) + np.testing.assert_array_equal(actual, expected) + np.testing.assert_array_equal(view, pixels) + if mask is not None: + np.testing.assert_array_equal(mask, mask_before) + assert not np.shares_memory(actual, view) + labels = np.zeros((11, 13), dtype=np.int32) + labels[2:8, 4:10] = 1 + indexes = np.array([1], dtype=np.int32) + expected_hull = centrosome.cpmorphology.convex_hull(labels, indexes) + actual_hull = label_geometry.CellProfilerLabelHull.from_labels(labels, indexes).vertices() + for actual, expected in zip(actual_hull, expected_hull): + np.testing.assert_array_equal(actual, expected) + endpoints = (np.array([0, 4]), np.array([0, -2]), np.array([4, 0]), np.array([2, 0])) + expected_lines = centrosome.cpmorphology.get_line_pts(*endpoints) + for actual, expected in zip(worm_geometry._cellprofiler_line_points(*endpoints), expected_lines): + np.testing.assert_array_equal(actual, expected) + assert actual.flags.owndata and actual.flags.writeable + assert tuple(tuple(dispatcher.signatures) for dispatcher in dispatchers) == before + """) + environment = os.environ.copy() + environment.update( + {"OPENHCS_CPU_ONLY": "true", "NUMBA_CACHE_DIR": str(tmp_path / "kernels")} + ) + result = subprocess.run( + (sys.executable, "-c", script), + cwd=Path(__file__).parents[2], + env=environment, + check=False, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/unit/test_cellprofiler_worm_geometry.py b/tests/unit/test_cellprofiler_worm_geometry.py index fae7fb7029..07d6742c60 100644 --- a/tests/unit/test_cellprofiler_worm_geometry.py +++ b/tests/unit/test_cellprofiler_worm_geometry.py @@ -76,3 +76,45 @@ def test_skeletonize_preserves_centrosome_top_border_behavior() -> None: skeletonize_worm_mask(np.ones((2, 2), dtype=bool)), np.array([[True, True], [False, False]]), ) + + +def test_line_points_match_centrosome_all_octants_and_keep_owned_outputs() -> None: + import centrosome.cpmorphology + + grid = np.arange(-3, 4) + end_rows, end_columns = np.meshgrid(grid, grid) + end_rows = end_rows.ravel() + end_columns = end_columns.ravel() + start_rows = np.zeros_like(end_rows) + start_columns = np.zeros_like(end_columns) + for endpoints in ( + (start_rows, start_columns, end_rows, end_columns), + (end_rows, end_columns, start_rows, start_columns), + tuple(np.empty(0, dtype=int) for _ in range(4)), + ): + expected = centrosome.cpmorphology.get_line_pts(*endpoints) + actual = _cellprofiler_line_points(*endpoints) + for result, oracle in zip(actual, expected): + np.testing.assert_array_equal(result, oracle) + assert result.dtype == np.dtype(int) + assert result.flags.owndata and result.flags.writeable + assert all(not np.shares_memory(result, source) for source in endpoints) + assert all( + not np.shares_memory(left, right) + for index, left in enumerate(actual) + for right in actual[index + 1 :] + ) + + +def test_line_points_reject_unequal_endpoint_lengths() -> None: + import pytest + + with pytest.raises( + ValueError, match="Line endpoint arrays must have equal lengths" + ): + _cellprofiler_line_points( + np.array([0, 1]), + np.array([0]), + np.array([2]), + np.array([2]), + ) From f70390176a1774115792e1721d9909b0badb4817 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 16:07:26 -0400 Subject: [PATCH 083/125] perf: bound dense moments and scan original strided geometry Use the existing dense ID-domain law on positive pixels before moment allocation, preserving sparse high-ID fallback and original domain callback order. Scan a shared strided 3D view rather than flatten-copying mutable label buffers; prepare all admitted layouts and mutability signatures before execution. Refs #384. --- openhcs/core/runtime_object_labels.py | 43 ++++++----- .../test_dense_object_coordinate_reduction.py | 73 +++++++++++++++++-- 2 files changed, 93 insertions(+), 23 deletions(-) diff --git a/openhcs/core/runtime_object_labels.py b/openhcs/core/runtime_object_labels.py index a503a78b14..396e677ae6 100644 --- a/openhcs/core/runtime_object_labels.py +++ b/openhcs/core/runtime_object_labels.py @@ -30,6 +30,7 @@ NominalTypeStrategyFamilyMixin, ) from openhcs.core.runtime_object_label_domains import ( + DenseIntegerObjectLabelIdDomain, ObjectLabelDomain, ObjectLabelDomainDeclaration, ObjectLabelDomainMetadata, @@ -1218,12 +1219,17 @@ def sparse_ijv_rows(self, labels: object) -> SparseIJVLabelRows: @classmethod def prepare_coordinates(cls) -> None: - """Prepare both mutability signatures of the dense coordinate reducer.""" - labels = np.array([[0, 1]], dtype=np.int32) - for writeable in (True, False): - labels.flags.writeable = writeable - _dense_label_coordinate_moments_numba(labels.ravel(), 1, 2, 1) - dense_label_centers_2d_numba(labels, 1) + """Prepare C/F/strided signatures with both input mutability policies.""" + plane = np.array([[0, 1], [1, 0]], dtype=np.int32) + volume = np.stack((plane, plane)) + for labels in (volume, np.asfortranarray(volume), volume.copy()[..., ::-1]): + for writeable in (True, False): + labels.flags.writeable = writeable + _dense_label_coordinate_moments_numba(labels, 1) + for labels in (plane, np.asfortranarray(plane), plane.copy()[:, ::-1]): + for writeable in (True, False): + labels.flags.writeable = writeable + dense_label_centers_2d_numba(labels, 1) def axis_centers( self, labels: object, *, domain: Sequence[int] @@ -1235,11 +1241,19 @@ def axis_centers( or any(size > np.iinfo(np.int32).max for size in array.shape) ): return super().axis_centers(labels, domain=domain) + coordinate_planes = array[None, ...] if array.ndim == 2 else array + positive_parts = tuple(plane[plane > 0] for plane in coordinate_planes) + coordinate_domain = DenseIntegerObjectLabelIdDomain.from_array( + np.concatenate(positive_parts or (np.empty(0, dtype=np.int32),)) + ) + del positive_parts + if coordinate_domain is None: + return super().axis_centers(labels, domain=domain) coordinate_columns = tuple(range(3 - array.ndim, 3)) sums, pixel_counts = _dense_label_coordinate_moments_numba( - array.ravel(order="C"), - array.shape[-2], array.shape[-1], int(array.max(initial=0)), + coordinate_planes, coordinate_domain.max_label, ) + del coordinate_domain # Like sparse conversion, the moments snapshot precedes domain callbacks. object_ids = np.flatnonzero(pixel_counts) max_domain_label = max(domain, default=0) @@ -1622,22 +1636,17 @@ def object_label_axis_centers( @njit(cache=True) def _dense_label_coordinate_moments_numba( - flat_labels: np.ndarray, - height: int, - width: int, + labels: np.ndarray, maximum_label: int, ) -> tuple[np.ndarray, np.ndarray]: """Reduce dense positive labels once in their declared row-major geometry.""" sums = np.zeros((maximum_label + 1, 3), dtype=np.float64) counts = np.zeros(maximum_label + 1, dtype=np.int64) - plane_size = height * width - plane_count = 0 if plane_size == 0 else flat_labels.size // plane_size - offset = 0 + plane_count, height, width = labels.shape for plane in range(plane_count): for y in range(height): for x in range(width): - label_id = int(flat_labels[offset]) - offset += 1 + label_id = int(labels[plane, y, x]) if label_id > 0 and label_id <= maximum_label: sums[label_id, 0] += plane sums[label_id, 1] += y @@ -1653,7 +1662,7 @@ def dense_label_centers_2d_numba( """Validate compiled two-dimensional geometry and return its y/x centers.""" height, width = labels.shape sums, counts = _dense_label_coordinate_moments_numba( - labels.ravel(), height, width, label_count + labels[None, :height, :width], label_count ) centers = np.empty((label_count + 1, 2), dtype=np.float64) for label_id in range(label_count + 1): diff --git a/tests/unit/test_dense_object_coordinate_reduction.py b/tests/unit/test_dense_object_coordinate_reduction.py index f3cc02442e..a136d5828c 100644 --- a/tests/unit/test_dense_object_coordinate_reduction.py +++ b/tests/unit/test_dense_object_coordinate_reduction.py @@ -198,6 +198,64 @@ def __iter__(self): assert not effects +@pytest.mark.parametrize("shape,label_id", (((1, 1), 2**31 - 1), ((64, 64), 4096))) +def test_sparse_high_ids_reach_domain_error_before_moment_allocation(monkeypatch, shape, label_id): + from openhcs.core import runtime_object_labels + + def reject_moment_allocation(*_args): + raise AssertionError("Sparse ID span must not allocate a dense moment table") + + monkeypatch.setattr(runtime_object_labels, "_dense_label_coordinate_moments_numba", reject_moment_allocation) + for call in (generic_centers, lambda labels, domain: object_label_axis_centers(labels, domain=domain)): + labels = np.zeros(shape, dtype=np.int32) + labels.flat[0] = label_id + effects = [] + + class Domain: + def __iter__(self): + effects.append("iterate") + labels.flat[0] = 1 + raise RuntimeError("Domain rejected sparse high IDs") + + with pytest.raises(RuntimeError, match="Domain rejected sparse high IDs"): + call(labels, Domain()) + assert effects == ["iterate"] + assert labels.flat[0] == 1 + + +@pytest.mark.parametrize("dimensions", (2, 3)) +@pytest.mark.parametrize("layout", ("fortran", "reversed", "broadcast")) +def test_coordinate_scan_borrows_strided_input_without_flatten_copy(monkeypatch, dimensions, layout): + from openhcs.core import runtime_object_labels + + labels = np.array([[0, 1, 1], [2, 0, 2]], dtype=np.int32) + if dimensions == 3: + labels = np.stack((labels, labels)) + if layout == "fortran": + labels = np.asfortranarray(labels) + elif layout == "reversed": + labels = labels[..., ::-1] + else: + labels = np.broadcast_to(labels[..., :1, :], (*labels.shape[:-2], 4, 3)) + expected = generic_centers(labels, (1, 2, 4)) + before = labels.copy() + original = runtime_object_labels._dense_label_coordinate_moments_numba + requests = [] + + def borrowed_scan(coordinate_planes, maximum_label): + assert coordinate_planes.ndim == 3 + assert np.shares_memory(coordinate_planes, labels) + assert coordinate_planes.flags.writeable == labels.flags.writeable + assert coordinate_planes.shape == ((1, *labels.shape) if dimensions == 2 else labels.shape) + requests.append(coordinate_planes) + return original(coordinate_planes, maximum_label) + + monkeypatch.setattr(runtime_object_labels, "_dense_label_coordinate_moments_numba", borrowed_scan) + assert_centers(object_label_axis_centers(labels, domain=(1, 2, 4)), expected) + assert len(requests) == 1 + np.testing.assert_array_equal(labels, before) + + def test_primary_objects_preparation_readies_all_admitted_storage_signatures(tmp_path): script = textwrap.dedent(""" from unittest.mock import patch @@ -214,15 +272,18 @@ def test_primary_objects_preparation_readies_all_admitted_storage_signatures(tmp signatures = tuple(tuple(kernel.signatures) for kernel in kernels) assert all(signatures) - def reject_compilation(signature): - raise AssertionError(('Late coordinate compilation', str(signature))) + def reject_compilation(*arguments): + raise AssertionError(('Late coordinate preparation', tuple(map(str, arguments)))) - with patch.object(_dense_label_coordinate_moments_numba, 'compile', - side_effect=reject_compilation), patch.object( - dense_label_centers_2d_numba, 'compile', side_effect=reject_compilation): + from contextlib import ExitStack + with ExitStack() as guards: + for kernel in kernels: + guards.enter_context(patch.object(kernel, 'compile', side_effect=reject_compilation)) + guards.enter_context(patch.object(kernel._cache, 'load_overload', side_effect=reject_compilation)) for shape in ((4, 5), (3, 4, 5)): base = np.arange(np.prod(shape), dtype=np.int32).reshape(shape) % 4 - for labels in (base, np.asfortranarray(base), base[..., ::-1], base.copy()): + for labels in (base, np.asfortranarray(base), base[..., ::-1], + np.broadcast_to(base[..., :1, :], shape), base.copy()): object_label_axis_centers(labels, domain=(1, 2, 3, 6)) labels.flags.writeable = False object_label_axis_centers(labels, domain=(1, 2, 3, 6)) From 94738b63d088444cfc99feb964f957bbc94d2e8e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 16:26:51 -0400 Subject: [PATCH 084/125] docs: qualify shared geometry and coordinate runtime gains (refs #384) --- ...shared-dense-coordinate-owner-20261002.rst | 111 ++++++++++++++++++ .../shared_runtime_plumbing_384_20261001.rst | 37 ++++++ 2 files changed, 148 insertions(+) create mode 100644 docs/validation/shared-dense-coordinate-owner-20261002.rst diff --git a/docs/validation/shared-dense-coordinate-owner-20261002.rst b/docs/validation/shared-dense-coordinate-owner-20261002.rst new file mode 100644 index 0000000000..4d865af45f --- /dev/null +++ b/docs/validation/shared-dense-coordinate-owner-20261002.rst @@ -0,0 +1,111 @@ +Shared dense coordinate reduction +================================ + +The shared reducer is integrated into PR394 for issue #384. The isolated +qualification source was ``d5441f960fb4cafe5a1f56740cf8c52de7a7d69d``, normally +merged with qualified PR394 ``eb2f23efd`` and main ``a6c18f054``. Integrated +controls use clean ``6e69b85122d3c1a44857835ba9b3fe77e743240d``. + +``ObjectLabelStorageStrategy`` retains the genuine sparse-IJV algorithm and +owns shared sum allocation/division. The existing dense leaf specializes int32 +2D/3D arrays; ``ObjectLabelValueStorageStrategy`` delegates actual storage. +Tracking and relationships consume one core numerical primitive through their +existing ABI adapters. Their two duplicate kernels are deleted, with all five +compiled consumers migrated. No new class, registry, dispatch table or cache +was introduced. + +Before a public domain iterator can mutate pixels or raise, the dense leaf +captures coordinate moments from a borrowing strided 3D view. The existing +``DenseIntegerObjectLabelIdDomain`` bounds moment allocation by the number of +positive pixels. Empty or large sparse ID spans use the original sparse path; +a lone ID ``2**31 - 1`` therefore reaches the original domain error without +allocating a max-ID-sized table. Unsupported dtypes/ranks retain original +conversion, warnings and errors. Readonly, reversed, Fortran and zero-stride +inputs avoid flatten copies. This establishes an O(nnz) allocation bound, not +identical heap thresholds under OOM or a resolution of issue #433. + +Existing provider/callable preparation warms C/F/A writable and readonly +signatures before execution. An isolated PrimaryObjects preparation control +refuses both compilation and cache loading during subsequent admitted calls. +Controls preserve effectful Sequence iteration, NumPy allocation-phase errors, +NaNs/counts, independent result buffers and input mutation/alias semantics. +Eleven effect controls also run against the actual unchanged Git parent. + +Qualification +------------- + +* Isolated synchronized controls: 100 passed, 2 existing skips. +* Integrated controls at ``6e69b85122``: 101 passed, 2 existing skips in + 194.57 seconds; tracked source hashes remain unchanged. +* Original scoped R0: every metric delta zero; original R1: no increases, + unchanged 160-second budget. Both compare ``eb2f23efd`` to ``d5441f960``. +* Global NRA census: 5,526 original classes, 5,513 canonical declarations, + 13 explicitly OPEN syntax rows, no parse errors. +* Saved real Wound pair: exact coordinates/counts and physical allocation/ + alias gates; original median 0.160669 s, candidate 0.019144 s, local saving + 0.141524 s across paired replay. Preparation/JIT precede timing. +* Twenty-one saved Track frames and old-Git tracking/relationship numerical + ABI comparisons pass. This is not a full relationship transaction proof. + +Initial admission, readiness and high-ID negative evidence remains retained. +These local results alone do not establish ordinary pipeline performance. +Root-owned ordinary qualification is recorded separately below. Illumination +has no centroid calls. + +Evidence: ``/var/tmp/openhcs-shared-dense-coordinate-owner-closure-v2-20261002.json``; +``/var/tmp/openhcs-shared-dense-production-saved-replay-v4-20261002/receipt.json``; +``/var/tmp/openhcs-pr394-integrated-dense-coordinate-controls-20261002.log``. + +Ordinary integrated comparison +------------------------------ + +Four uninstrumented public sweeps in ABBA order compare frozen b4700288f +(shared geometry baseline) with integrated 6e69b8512 (shared coordinate owner). +Each case has one well, inline worker and thread on CPU5, default OUTCOMES +and memory observer, and a reused READY server. Server/library/kernel startup +and shutdown are excluded; compilation remains in total. No samples are removed. + +Two-sample descriptive means (seconds; not statistical significance): + +* ExampleWoundHealing: execution 3.287030339 -> 2.984141707; + compilation 0.337689161 -> 0.340739846; + total 3.866369809 -> 3.562642806. +* ExampleTrackObjects: execution 5.338271976 -> 5.052027345; + compilation 0.929434419 -> 1.252314091; + total 6.629491309 -> 6.684317434. + +Track total regresses by 0.054826125 seconds in this comparison. Its compilation +increases by 0.322879672 seconds; no causal late-JIT attribution is established. +Wound total decreases by 0.303727004 seconds. Neither result satisfies the +remaining whole-pipeline objective or replaces the generic-plumbing frontier. + +Every observation, in case/compile/execution/total seconds: + +* 0-A ExampleTrackObjects: 0.950359821, + 5.625208855, 6.914708113. +* 0-A ExampleWoundHealing: 0.343463182, + 3.276733875, 3.827193568. +* 1-B ExampleTrackObjects: 1.421878338, + 5.020748138, 6.829343345. +* 1-B ExampleWoundHealing: 0.342035055, + 3.044292212, 3.623388787. +* 2-B ExampleTrackObjects: 1.082749844, + 5.083306551, 6.539291524. +* 2-B ExampleWoundHealing: 0.339444637, + 2.923991203, 3.501896825. +* 3-A ExampleTrackObjects: 0.908509016, + 5.051335096, 6.344274506. +* 3-A ExampleWoundHealing: 0.331915140, + 3.297326803, 3.905546051. + +All eight complete authored-output inventories, strict table/image comparisons, +scientific file bytes and source/environment/native/input freezes pass. Wound +retains full native qualification through the exact qualified output reference. +Track preserves all original OH bytes and its original 77,726-pixel native RED; +this does not include the separate draft renderer repair in PR468. No parity +tolerance, exclusion or output policy changes. + +Receipt: ``/var/tmp/openhcs-pr394-shared-dense-ordinary-abba-science-20261002.json`` +SHA256 ``affb93c5955585be6963936552a1f40da051989688e7b0c58f406f398e894400``. +Commands: ``run_pr394_shared_dense_ordinary_abba_20261002.py`` and +``validate_pr394_shared_dense_ordinary_abba_science_20261002.py`` under /var/tmp. diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 16cf3492e0..b32bf532b1 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1742,3 +1742,40 @@ prepared-Resize consumer. The original whole-branch scalar-classifier R0 RED remains separately retained. These draft closing references do not establish completion. Full native scaling, all30 benchmark reruns and fresh figures remain outstanding, and the long-running performance goal stays active. + +Shared geometry release and coordinate-owner checkpoint +------------------------------------------------------ + +Main ``5b7f3c48d89da4415de001375d0e06de881153c9`` is normally merged. +Independent PR476 ships the shared quantized hull/Bresenham geometry and +officially closes issue475. Earlier hull issues158/268 and PR161 remain +completed. The full-source PR394 ABBA comparison pins eb2 versus b470: +BrightField total4.635292249 ->3.294145465s, Untangle3.234328632 ->2.311149861s, +Illumination1.988957128 ->1.892348221s. All16 complete saved outputs pass exact +science and source/env/native/input freezes. Wound is a variance control, not +an attributed hull saving. See the committed shared-quantized-hull report for +every sample, readiness/source guards and retained native-image RED. + +The existing object-label storage family now owns the single shared dense +coordinate reducer. Tracking and relationships inherit/use its primitive; +two duplicate kernels are removed. Resource/domain-effect and C/F/A readiness +controls pass. Isolated100 and integrated101 tests pass, both with2 existing +skips; original scoped R0/R1 pass unchanged against eb2. Frozen b470 versus +6e69 ordinary ABBA Wound total3.866369809 ->3.562642806s; Track execution +5.338271976 ->5.052027345s, but total6.629491309 ->6.684317434s regresses +with higher compilation. All8 full saved outputs/freeze gates pass. The separate +shared-dense-coordinate report retains all samples, no causal late-JIT claim +and the original tracking image RED. No reduction resolves issue433's original +allocation owner or the whole-branch scalar-classifier R0 RED. + +Owner WT remains /home/ts/code/projects/openhcs-shared-runtime-plumbing. All +production edits are committed here. Agent-owned implementation branches are +integrated; no additional unpublished production files are claimed. Frozen +benchmark WTs remain unchanged. Compared with receiving checkpoint eb2, new +production edits are precisely three CP geometry/illumination modules plus +core/runtime_object_labels.py and four CP backend/preparation/tracking/ +relationships modules. These are indirect numerical/coordinate and preparation +changes beyond the receiving review's original four-file eb2/wheel delta. +Installed receiving obligations419/433/435/450, whole-branch R0, full native +image parity, full-catalog/scaling and figures remain open. A first-wave fresh +four-case ordinary/native qualification is in progress, not a completed gate. From b25120d5a91f6d4f40e9742982ab143667e91398 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 16:43:27 -0400 Subject: [PATCH 085/125] Project match-image labels on declared scalar singleton invocations (refs #477) --- .../cellprofiler/runtime/module_execution.py | 19 +++ ...test_singleton_rgb_label_invocation_477.py | 131 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 tests/unit/test_singleton_rgb_label_invocation_477.py diff --git a/openhcs/interop/cellprofiler/runtime/module_execution.py b/openhcs/interop/cellprofiler/runtime/module_execution.py index 4507007dd4..180527d4bc 100644 --- a/openhcs/interop/cellprofiler/runtime/module_execution.py +++ b/openhcs/interop/cellprofiler/runtime/module_execution.py @@ -36,6 +36,7 @@ MeasurementSparseColumnarRows, ) from openhcs.core.pipeline.function_contracts import ( + ObjectLabelInputExecutionMode, object_label_input_execution_mode_from_callable, ) from openhcs.core.runtime_adapters import ( @@ -1494,6 +1495,24 @@ def _invocation_request( kwargs=runtime_kwargs, variable_components=adapter.request.variable_components, ) + # A scalar image can result from composing channels on a declared + # singleton runtime root. Match-image labels consume that root only + # after the module has chosen its final image domain and execution mode. + if ( + object_label_input_execution_mode_from_callable(self.raw_func) + is ObjectLabelInputExecutionMode.MATCH_IMAGE_STACK + and execution_mode is ImagePayloadExecutionMode.NATURAL + and image_request.plane_projection is None + and runtime_projection is not None + and runtime_projection.axis_size == 1 + ): + runtime_kwargs = cast( + dict[str, RuntimeCallableArgument], + RuntimeSliceProjection.kwargs_for_slice( + runtime_kwargs, + runtime_projection.selected_plane(0), + ), + ) if profile_enabled: CellProfilerRuntimeProfileLogger.log_module_profile( "cp_invocation_execution_mode_policy", diff --git a/tests/unit/test_singleton_rgb_label_invocation_477.py b/tests/unit/test_singleton_rgb_label_invocation_477.py new file mode 100644 index 0000000000..9b9354338f --- /dev/null +++ b/tests/unit/test_singleton_rgb_label_invocation_477.py @@ -0,0 +1,131 @@ +"""Scalar RGB invocations consume only an explicitly declared singleton root.""" +import numpy as np +import pytest + +from openhcs.core.artifacts import ArtifactSpec, ImageArtifactType, ObjectLabelsArtifactType +from openhcs.core.aligned_image_payload import ImagePayloadExecutionMode +from openhcs.core.function_contract_metadata import FunctionContractAttribute +from openhcs.core.pipeline.function_contracts import ObjectLabelInputExecutionMode +from openhcs.core.runtime_image_values import ImageMetadataPayload, ImagePayloadMetadata +from openhcs.core.runtime_object_label_domains import ObjectLabelDomain, ObjectLabelDomainScope +from openhcs.core.runtime_object_labels import ObjectLabelSet, ObjectLabelVariantData, object_label_dense_array +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis, RuntimePlaneProjection +from openhcs.core.runtime_slice_projection import RuntimeSliceProjectionDeclarationError +from openhcs.interop.cellprofiler.runtime.function_contract_execution import CellProfilerFunctionContractExecutor +from openhcs.processing.backends.cellprofiler.outlines import OverlayOutlinesModule +from tests.unit.test_cellprofiler_module_execution import ( + _FakeCellProfilerRuntime, + _artifact_input_edge_for_test, + _compiled_callable_contract, + _module_executor, +) + + +def _invocation_case(*, root_count=1, label_count=1): + image_spec = ArtifactSpec.input('GFPandDNA', ImageArtifactType) + labels_spec = ArtifactSpec.input('Nuclei', ObjectLabelsArtifactType) + raw = OverlayOutlinesModule.require_callable() + contract = _compiled_callable_contract(raw, artifact_inputs=(image_spec, labels_spec)) + data = np.linspace(0, 1, 8 * 9 * 3, dtype=np.float32).reshape(8, 9, 3) + image = ImageMetadataPayload(data, ImagePayloadMetadata(source_channel_axis=2, source_image_names=('GFPandDNA',))) + label_data = np.zeros((label_count, 8, 9), dtype=np.int32) + label_data[:, 2:5, 3:6] = 1 + labels = ObjectLabelSet( + name='Nuclei', variant_data=ObjectLabelVariantData(labels=label_data), + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + domain=ObjectLabelDomain(declared_object_id_domains=((1,),) * label_count, scope=ObjectLabelDomainScope.PLANE), + ) + adapter = _FakeCellProfilerRuntime( + {'GFPandDNA': image}, objects={'Nuclei': labels}, callable_contract=contract, + artifact_input_edges=tuple(_artifact_input_edge_for_test(spec) for spec in (image_spec, labels_spec)), + plane_projection=RuntimePlaneProjection.stack(root_count), + ) + executor = _module_executor(contract) + request = executor._image_request(image, adapter, module_type=OverlayOutlinesModule, active_input_specs=(image_spec, labels_spec)) + return raw, contract, image, labels, adapter, executor, request + + +def _bind(case): + raw, contract, image, labels, adapter, executor, request = case + return executor._invocation_request( + image_request=request, adapter=adapter, current_image=image, + kwargs={'outline_colors': ('#21FFFF',)}, module_type=OverlayOutlinesModule, + ) + + +def _execute(case, invocation): + raw, contract, *_ = case + return CellProfilerFunctionContractExecutor().execute( + contract, raw, invocation.image, invocation.kwargs, + execution_mode=invocation.execution_mode, plane_projection=invocation.plane_projection, + ) + + +def test_match_image_labels_consume_declared_singleton_root_for_scalar_rgb(): + case = _invocation_case() + raw, contract, image, labels, *_ = case + image_before = image.data.copy() + labels_before = object_label_dense_array(labels).copy() + invocation = _bind(case) + selected = invocation.kwargs['object_labels'][0] + assert invocation.execution_mode is ImagePayloadExecutionMode.NATURAL + assert invocation.plane_projection is None + assert invocation.image.metadata.source_channel_axis == 2 + assert selected.plane_axis is None + assert object_label_dense_array(selected).shape == (8, 9) + output = _execute(case, invocation) + scalar_labels = ObjectLabelSet(name='Nuclei', variant_data=ObjectLabelVariantData(labels=labels_before[0]), domain=ObjectLabelDomain(declared_object_ids=(1,), scope=ObjectLabelDomainScope.PAYLOAD)) + expected = raw(image, object_labels=(scalar_labels,), outline_colors=('#21FFFF',)) + np.testing.assert_array_equal(output.data, expected.data) + np.testing.assert_array_equal(image.data, image_before) + np.testing.assert_array_equal(object_label_dense_array(labels), labels_before) + assert labels.plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert image.metadata.plane_axis is None + + +@pytest.mark.parametrize('root_count', (None, 2)) +def test_scalar_rgb_cannot_authorize_unknown_or_multiple_root_planes(root_count): + case = _invocation_case(root_count=root_count) + invocation = _bind(case) + assert invocation.kwargs['object_labels'][0].plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + with pytest.raises(RuntimeSliceProjectionDeclarationError, match='Kwargs cannot create image-axis execution semantics'): + _execute(case, invocation) + + +def test_explicit_full_stack_label_contract_preserves_scalar_boundary_refusal(monkeypatch): + raw = OverlayOutlinesModule.require_callable() + monkeypatch.setattr(raw, FunctionContractAttribute.object_label_input_execution_mode, ObjectLabelInputExecutionMode.FULL_STACK) + case = _invocation_case() + invocation = _bind(case) + assert invocation.kwargs['object_labels'][0].plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + with pytest.raises(RuntimeSliceProjectionDeclarationError, match='Kwargs cannot create image-axis execution semantics'): + _execute(case, invocation) + + +def test_declared_singleton_root_refuses_mismatched_label_cardinality(): + case = _invocation_case(label_count=2) + with pytest.raises(ValueError, match='Object-label runtime plane-axis cardinality mismatch'): + _bind(case) + + +def test_final_module_mode_sees_full_labels_before_scalar_projection(monkeypatch): + case = _invocation_case() + def mode(cls, default, *, image, kwargs, variable_components): + assert kwargs['object_labels'][0].plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + assert object_label_dense_array(kwargs['object_labels'][0]).shape == (1, 8, 9) + return ImagePayloadExecutionMode.FULL_STACK + monkeypatch.setattr(OverlayOutlinesModule, 'execution_mode', classmethod(mode)) + invocation = _bind(case) + assert invocation.execution_mode is ImagePayloadExecutionMode.FULL_STACK + assert invocation.kwargs['object_labels'][0].plane_axis is RuntimePlaneAxis.RUNTIME_SLICE + + +def test_module_mode_failure_precedes_projection_guard(monkeypatch): + case = _invocation_case(label_count=2) + error = RuntimeError('module image-domain selection failed') + def mode(cls, default, *, image, kwargs, variable_components): + raise error + monkeypatch.setattr(OverlayOutlinesModule, 'execution_mode', classmethod(mode)) + with pytest.raises(RuntimeError) as caught: + _bind(case) + assert caught.value is error From ccaa4cba6a1eca818cd544438941f1c1cedb3e32 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 16:47:18 -0400 Subject: [PATCH 086/125] Own singleton label projection on the declared input policy (refs #477) --- openhcs/core/pipeline/function_contracts.py | 32 +++++++++++++- .../cellprofiler/runtime/module_execution.py | 43 ++++++------------- 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/openhcs/core/pipeline/function_contracts.py b/openhcs/core/pipeline/function_contracts.py index 413f266de8..e0d343fc5f 100644 --- a/openhcs/core/pipeline/function_contracts.py +++ b/openhcs/core/pipeline/function_contracts.py @@ -1,6 +1,6 @@ """Function-level artifact contract decorators for the pipeline compiler.""" -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import replace from enum import Enum import inspect @@ -33,6 +33,8 @@ ImagePayloadConsumption, ) from openhcs.core.function_contract_metadata import FunctionContractAttribute +from openhcs.core.image_payload_execution_mode import ImagePayloadExecutionMode +from openhcs.core.runtime_plane_projection import RuntimePlaneAxisValueProjection from openhcs.core.variable_component_stack_requirement import ( AlwaysRequiresVariableComponentStack, VariableComponentStackRequirement, @@ -136,6 +138,34 @@ def preserves_full_stack(self, *, image_stack_required: bool) -> bool: self is self.MATCH_IMAGE_STACK and image_stack_required ) + def invocation_kwargs( + self, + kwargs: Mapping[str, Any], + *, + execution_mode: ImagePayloadExecutionMode, + image_projection: RuntimePlaneAxisValueProjection | None, + runtime_projection: RuntimePlaneAxisValueProjection | None, + semantic_controls: Mapping[str, Any], + ) -> dict[str, Any]: + """Match scalar images to their declared singleton runtime root. + + Explicit full-stack label consumers keep their domain. Matching labels + can consume a singleton root only after the image's final execution + mode and retained plane projection have been resolved. + """ + if ( + self is self.MATCH_IMAGE_STACK + and execution_mode is ImagePayloadExecutionMode.NATURAL + and image_projection is None + ): + if runtime_projection is not None and runtime_projection.axis_size == 1: + from openhcs.core.runtime_slice_projection import RuntimeSliceProjection + + kwargs = RuntimeSliceProjection.kwargs_for_slice( + kwargs, runtime_projection.selected_plane(0) + ) + return {**kwargs, **semantic_controls} + def _artifact_spec_from_output_declaration( spec: str | ArtifactSpec | tuple[str, MaterializationSpec | None], diff --git a/openhcs/interop/cellprofiler/runtime/module_execution.py b/openhcs/interop/cellprofiler/runtime/module_execution.py index 180527d4bc..170b24ac37 100644 --- a/openhcs/interop/cellprofiler/runtime/module_execution.py +++ b/openhcs/interop/cellprofiler/runtime/module_execution.py @@ -36,7 +36,6 @@ MeasurementSparseColumnarRows, ) from openhcs.core.pipeline.function_contracts import ( - ObjectLabelInputExecutionMode, object_label_input_execution_mode_from_callable, ) from openhcs.core.runtime_adapters import ( @@ -1484,50 +1483,32 @@ def _invocation_request( image_request=image_request, runtime_kwargs=runtime_kwargs, ) - invocation_image = image_request.payload - default_execution_mode = ( - self.callable_contract.runtime_image_execution_mode - or image_request.execution_mode - ) execution_mode = module_type.execution_mode( - default_execution_mode, - image=invocation_image, + self.callable_contract.runtime_image_execution_mode + or image_request.execution_mode, + image=image_request.payload, kwargs=runtime_kwargs, variable_components=adapter.request.variable_components, ) - # A scalar image can result from composing channels on a declared - # singleton runtime root. Match-image labels consume that root only - # after the module has chosen its final image domain and execution mode. - if ( - object_label_input_execution_mode_from_callable(self.raw_func) - is ObjectLabelInputExecutionMode.MATCH_IMAGE_STACK - and execution_mode is ImagePayloadExecutionMode.NATURAL - and image_request.plane_projection is None - and runtime_projection is not None - and runtime_projection.axis_size == 1 - ): - runtime_kwargs = cast( - dict[str, RuntimeCallableArgument], - RuntimeSliceProjection.kwargs_for_slice( - runtime_kwargs, - runtime_projection.selected_plane(0), - ), - ) if profile_enabled: CellProfilerRuntimeProfileLogger.log_module_profile( "cp_invocation_execution_mode_policy", time.perf_counter() - phase_started_at, module=module_name, ) - invocation_kwargs = { - **runtime_kwargs, - **_execution_mode_semantic_control_kwargs( + invocation_kwargs = object_label_input_execution_mode_from_callable( + self.raw_func + ).invocation_kwargs( + runtime_kwargs, execution_mode=execution_mode, + image_projection=image_request.plane_projection, + runtime_projection=runtime_projection, + semantic_controls=_execution_mode_semantic_control_kwargs( self.callable_contract.require_processing_contract(), execution_mode, ), - } + ) return RuntimeFunctionInvocationRequest( - image=invocation_image, + image=image_request.payload, kwargs=invocation_kwargs, source_image_name=image_request.source_image_name, image_count=image_request.image_count, From 619468b998482b6e0cbcf212f8f8e1e02a9fbef6 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:08:03 -0400 Subject: [PATCH 087/125] docs: qualify generic singleton label invocation repair (refs #477) --- ...ingleton-label-invocation-477-20261002.rst | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/validation/singleton-label-invocation-477-20261002.rst diff --git a/docs/validation/singleton-label-invocation-477-20261002.rst b/docs/validation/singleton-label-invocation-477-20261002.rst new file mode 100644 index 0000000000..faad1c18a4 --- /dev/null +++ b/docs/validation/singleton-label-invocation-477-20261002.rst @@ -0,0 +1,92 @@ +Singleton label projection at the final invocation boundary (#477) +================================================================= + +The ordinary translocation benchmark failed on frozen ``6e69b851`` before +producing a successful well. Its OverlayOutlines invocation paired a scalar +640-by-640 RGB image with object labels on a declared singleton runtime root. +The image had a channel axis but no image-plane projection. Treating the RGB +channels as image planes would violate the existing strict axis contract. + +The input binding request initially derived label shape with the callable's +FLEXIBLE default. The module executor resolved the final NATURAL mode later, +after binding, so its semantic controls and the bound labels could disagree. + +Existing nominal ownership +------------------------- + +``ObjectLabelInputExecutionMode.invocation_kwargs`` now owns this final +projection. MATCH_IMAGE_STACK with a NATURAL scalar image may project an +already-bound label argument only when the original runtime declaration proves +exactly one plane. It delegates to the existing ``RuntimeSliceProjection`` and +merges the resolved semantic controls once. The module executor delegates to +that owner after resolving execution mode. + +Explicit FULL_STACK label declarations and FULL_STACK execution remain intact. +Multiple or unproved runtime axes and retained image-plane projections keep +their original strict guards. The fix neither invents an image axis nor reloads +artifacts. Redundant intermediate image/default-mode locals are removed. No +new policy class, cache, per-module exception, or color-processing change is +introduced. + +Source and local validation +--------------------------- + +The final isolated source is ``ccaa4cba6a1eca818cd544438941f1c1cedb3e32``, +integrated into PR #394 by ``1763ef07eba959dbefa7643cb90c8230fdf12518``. +The initial inline implementation and its original R0 failure are retained in +history; that route added GodClass, boolean-chain and foreign-absence metrics. +The final existing-owner implementation passes unchanged original R0 metrics +and original R1 with its original 160-second budget. + +The original reproducer recorded two expected failures and five passing +controls. The repaired module-consumer gate passes 402 controls. The global +source census parsed 703 modules with no errors. All five registered +MATCH_IMAGE_STACK consumers currently declare FLEXIBLE processing contracts. +External custom MATCH consumers with a non-FLEXIBLE stack-only contract have +not been qualified; this report makes no blanket callback-equivalence claim. + +The source-qualified receipt is +``/var/tmp/issue477-policy-owner-qualified-receipt-20261002.json`` with SHA256 +``7ebcd89739bbe58cf7630c093f02b92ecf41732a196073641ec373e4d9b019ca``. + +Actual public pipeline consumer +------------------------------- + +The frozen integrated source at +``/var/tmp/openhcs-pr394-singleton-label-qualified-20261002`` remains immutable. +Its ordinary public run completed both translocation and WoundHealing with one +successful well each, a reused READY server, one inline worker and one thread +on CPU 5. All input, native-binary, environment, dependency and tracked-source +freezes pass. Server/library/kernel preparation and shutdown are outside the +pipeline clocks; compilation and OUTCOMES closure remain inside total. + +The single observations are: + +================= ============== ============== ============== +Case Compilation(s) Execution(s) Total(s) +================= ============== ============== ============== +Translocation 0.527004 1.570796 2.539227 +WoundHealing 1.296431 2.897727 4.421997 +================= ============== ============== ============== + +These are correctness observations, not matched optimization evidence. +Wound's higher compilation observation is retained and its cause is unknown. +The original failed translocation clocks remain excluded from speedup claims. +The observation receipt is +``/var/tmp/openhcs-pr394-singleton-label-ordinary-v1-20261002/observations.json``; +its source-freeze SHA256 is +``ea4969dbfe48e7a42ff039b07a32eb0d2464374362b89588682483fa33039128``. + +Wound's complete authored output inventory, table comparison and exact +scientific bytes pass against the qualified ``9aa3918d`` reference. Its +two-row Image.csv SHA256 remains +``939735fa17a067137fea2adc77d57f9ce1df24bfcb6588c49023de1cc3998677``. +The read-only scientific receipt is +``/var/tmp/openhcs-pr394-singleton-label-wound-science-20261002.json`` with SHA256 +``7a8c3cafa33e54394d9a24f7744c6a6b641e48c6f712f1baab7c632dcace0655``. + +Translocation still requires fresh warm-native execution and complete authored +table, image, discrete-value and inventory comparison before issue acceptance. +The first native-controller preparation correctly rejected an auxiliary CSV +misclassified as an image input; its failure is retained. No tolerance change, +scientific-output exclusion or issue-closure claim is made here. From a73f48f0d3308b3a2763bb0d8d8312caf2c99d6e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:28:15 -0400 Subject: [PATCH 088/125] Remove identical singleton invocation method introduced by main merge --- openhcs/core/pipeline/function_contracts.py | 29 --------------------- 1 file changed, 29 deletions(-) diff --git a/openhcs/core/pipeline/function_contracts.py b/openhcs/core/pipeline/function_contracts.py index 50fc409f65..e0d343fc5f 100644 --- a/openhcs/core/pipeline/function_contracts.py +++ b/openhcs/core/pipeline/function_contracts.py @@ -167,35 +167,6 @@ def invocation_kwargs( return {**kwargs, **semantic_controls} - def invocation_kwargs( - self, - kwargs: Mapping[str, Any], - *, - execution_mode: ImagePayloadExecutionMode, - image_projection: RuntimePlaneAxisValueProjection | None, - runtime_projection: RuntimePlaneAxisValueProjection | None, - semantic_controls: Mapping[str, Any], - ) -> dict[str, Any]: - """Match scalar images to their declared singleton runtime root. - - Explicit full-stack label consumers keep their domain. Matching labels - can consume a singleton root only after the image's final execution - mode and retained plane projection have been resolved. - """ - if ( - self is self.MATCH_IMAGE_STACK - and execution_mode is ImagePayloadExecutionMode.NATURAL - and image_projection is None - ): - if runtime_projection is not None and runtime_projection.axis_size == 1: - from openhcs.core.runtime_slice_projection import RuntimeSliceProjection - - kwargs = RuntimeSliceProjection.kwargs_for_slice( - kwargs, runtime_projection.selected_plane(0) - ) - return {**kwargs, **semantic_controls} - - def _artifact_spec_from_output_declaration( spec: str | ArtifactSpec | tuple[str, MaterializationSpec | None], ) -> ArtifactSpec: From a8953b97ee8b61f736962f8729cdf9ea299f0e6b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:30:56 -0400 Subject: [PATCH 089/125] fix(equivalence): preserve saved relationship correlations (refs #479) --- benchmark/matched_cellprofiler_batch.py | 48 +- openhcs/core/equivalence/relationships.py | 463 +++++++++++++++++- openhcs/core/runtime_equivalence.py | 156 ++++-- openhcs/core/runtime_relationships.py | 159 +++++- .../cellprofiler/measurement_lookup.py | 29 +- .../runtime/relationship_measurement_rows.py | 48 +- .../backends/cellprofiler/relationships.py | 4 +- .../test_cellprofiler_module_execution.py | 6 +- ...est_cellprofiler_relationship_contracts.py | 8 +- .../test_exported_relationship_equivalence.py | 393 +++++++++++++++ 10 files changed, 1187 insertions(+), 127 deletions(-) create mode 100644 tests/unit/test_exported_relationship_equivalence.py diff --git a/benchmark/matched_cellprofiler_batch.py b/benchmark/matched_cellprofiler_batch.py index 53441d6777..77845e4437 100644 --- a/benchmark/matched_cellprofiler_batch.py +++ b/benchmark/matched_cellprofiler_batch.py @@ -179,14 +179,51 @@ def _require_compared_output_inventory( candidate_managed_files: frozenset[Path] = frozenset(), ) -> None: """Reject unqualified output formats and missing scientific output files.""" + # Validate all explicit edges against their object rows, then compare the + # cross-output correlations before using redundant scalar table inventory. + from openhcs.core.equivalence.relationships import ( + ExportedRelationshipMeasurementSemantics, + ) + from openhcs.core.runtime_equivalence import RuntimeMeasurementSnapshot + + measurements = tuple( + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + for snapshot in (reference_snapshot, candidate_snapshot) + ) + if any( + measurement.correlated_relationships is None for measurement in measurements + ): + raise RuntimeError( + "Matched saved output inventory requires known relationship correlations." + ) + if measurements[0].relationship_differences( + measurements[1].correlated_relationships + ): + raise RuntimeError("Matched saved output relationship correlations differ.") + comparable_tables = tuple( + tuple( + table + for table in snapshot.tables + if table.participates_in_comparison + and not ExportedRelationshipMeasurementSemantics.supports_table(table) + ) + for snapshot in (reference_snapshot, candidate_snapshot) + ) counts = [] - for files, exports, snapshot, managed_files in ( - (reference_files, reference_exports, reference_snapshot, frozenset()), + for files, exports, snapshot, managed_files, tables in ( + ( + reference_files, + reference_exports, + reference_snapshot, + frozenset(), + comparable_tables[0], + ), ( candidate_files, candidate_exports, candidate_snapshot, candidate_managed_files, + comparable_tables[1], ), ): files = files - managed_files @@ -210,7 +247,7 @@ def _require_compared_output_inventory( - len(snapshot.tables) - len(exports.image_outputs) + len(snapshot.images) - + sum(table.participates_in_comparison for table in snapshot.tables) + + len(tables) ) if count < 1: raise RuntimeError("Matched batch has no compared scientific output files.") @@ -232,10 +269,9 @@ def _require_compared_output_inventory( ), len(table.rows), ) - for table in snapshot.tables - if table.participates_in_comparison + for table in tables ) - for snapshot in (reference_snapshot, candidate_snapshot) + for tables in comparable_tables ) if table_shapes[0] != table_shapes[1]: raise RuntimeError( diff --git a/openhcs/core/equivalence/relationships.py b/openhcs/core/equivalence/relationships.py index 1f03d975c8..6b77b7717f 100644 --- a/openhcs/core/equivalence/relationships.py +++ b/openhcs/core/equivalence/relationships.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass, field -from typing import ClassVar +from typing import ClassVar, Self from metaclass_registry import RegistryFamily, RegistryKeyAttribute import numpy as np @@ -54,7 +54,7 @@ RuntimeMeasurementDialect, normalize_runtime_identifier, ) -from openhcs.core.equivalence.tables import measurement_table_padding_group +from openhcs.core.equivalence.tables import RuntimeTableSnapshot, measurement_table_padding_group from openhcs.core.registry_strategies import MostDerivedContextStrategyMixin from openhcs.core.measurement_row_materialization import ( columnar_row_values, @@ -66,9 +66,15 @@ MeasurementScope, MeasurementStatistic, ObjectCoreMeasurementFeature, + MeasurementScalarLiteral, ) from openhcs.core.runtime_relationships import ( ObjectInstanceKey, + ObjectRelationshipDeclaration, + DirectedObjectRelationshipPayload, + DirectParentReferenceFeatureDeclaration, + ChildCountFeatureDeclaration, + DirectParentReferenceMeasurementFeature, ObjectInstanceRelationship, ) from openhcs.core.runtime_tabular_values import ( @@ -1047,3 +1053,456 @@ def mean_child_value( if not values: return float("nan") return float(sum(values) / len(values)) + + +@dataclass(frozen=True, slots=True) +class ExportedRelationshipMeasurementSemantics(RelationshipMeasurementSemantics): + """Restore directed edge meaning using the saved object-row domains.""" + + object_tables: tuple[RuntimeScopedMeasurementTable, ...] + image_number_offset: RuntimeImageNumberOffset + + @classmethod + def supports_table(cls, table: RuntimeTableSnapshot) -> bool: + fields = frozenset(normalize_runtime_identifier(name) for name in table.header) + declaration_fields = frozenset( + ObjectRelationshipDeclaration.exported_field_names + ) + if not fields & declaration_fields: + return False + if not declaration_fields <= fields: + raise ValueError( + f"Relationship table {table.path} has an incomplete declaration schema." + ) + return True + + @classmethod + def validated_output_tables( + cls, + tables: tuple[RuntimeTableSnapshot, ...], + policy: RuntimeEquivalencePolicy, + ) -> tuple[RuntimeTableSnapshot, ...]: + """Consume every edge table before exposing its redundant scalar projection.""" + ordinary = tuple(table for table in tables if not cls.supports_table(table)) + object_tables = tuple( + RuntimeScopedMeasurementTable(measurement) + for table in ordinary + for measurement in table.measurement_tables(policy.measurement_dialect) + ) + image_offset = RuntimeImageNumberOffset.from_runtime_rows( + row + for table in object_tables + for row in table.table.rows.iter_row_mappings() + ) + relationships = tuple( + relationship + for table in tables + if cls.supports_table(table) + for relationship in cls.from_table( + table, object_tables, policy, image_offset + ) + ) + for relationship in relationships: + relationship.validate_recorded_measurements() + relationship.validate_reciprocal_declarations(relationships) + return ordinary + + @classmethod + def from_table( + cls, + table: RuntimeTableSnapshot, + object_tables: tuple[RuntimeScopedMeasurementTable, ...], + policy: RuntimeEquivalencePolicy, + image_number_offset: RuntimeImageNumberOffset, + ) -> tuple[Self, ...]: + if not table.rows: + raise ValueError("Relationship table has no determining declaration rows.") + normalized_header = tuple( + normalize_runtime_identifier(name) for name in table.header + ) + if len(set(normalized_header)) != len(normalized_header): + raise ValueError("Relationship table has duplicate normalized columns.") + groups: dict[ + ObjectRelationshipDeclaration, list[RuntimeMeasurementRowMapping] + ] = {} + for values in table.rows: + row = RuntimeMeasurementRowMapping( + dict(zip(table.header, values, strict=True)) + ) + declaration = ObjectRelationshipDeclaration.from_exported_values( + *( + row.first_value((field,)) + for field in ObjectRelationshipDeclaration.exported_field_names + ) + ) + groups.setdefault(declaration, []).append(row) + results = [] + for declaration, rows in groups.items(): + source_ids = [] + target_ids = [] + slice_indices = [] + counts = set() + for row in rows: + source_id = MeasurementScalarLiteral( + row.first_value((declaration.source_id_field,)) + ).integer_value + target_id = MeasurementScalarLiteral( + row.first_value((declaration.target_id_field,)) + ).integer_value + if ( + source_id is None + or target_id is None + or min(source_id, target_id) < 0 + ): + raise ValueError( + "Relationship endpoint IDs must be nonnegative integers." + ) + slice_index = image_number_offset.object_instance_key( + row.row, 1 + ).slice_index + if slice_index is None: + raise ValueError( + "Relationship rows require an explicit image or slice identity." + ) + raw_count = row.first_value(("slice_count",)) + count = MeasurementScalarLiteral(raw_count).integer_value + if raw_count is not None and count is None: + raise ValueError( + "Relationship slice count must be an integer when declared." + ) + counts.add(count) + source_ids.append(source_id) + target_ids.append(target_id) + slice_indices.append(slice_index) + if len(counts) != 1: + raise ValueError( + "Relationship rows have conflicting declared slice counts." + ) + relationship = ObjectRelationship( + name=declaration.artifact_name(), + declaration=declaration, + payload=DirectedObjectRelationshipPayload( + source_ids=tuple(source_ids), + target_ids=tuple(target_ids), + slice_indices=tuple(slice_indices), + slice_count=counts.pop(), + ), + ) + results.append( + cls( + relationship, + policy.measurement_dialect, + object_tables, + image_number_offset, + ) + ) + return tuple(results) + + @property + def source_name(self) -> str: + declaration = self.relationship.declaration + ref = ( + declaration.source + if declaration.projects_parent_child_measurements() + else declaration.target + ) + return normalize_runtime_identifier(ref.name) + + @property + def target_name(self) -> str: + declaration = self.relationship.declaration + ref = ( + declaration.target + if declaration.projects_parent_child_measurements() + else declaration.source + ) + return normalize_runtime_identifier(ref.name) + + @property + def instance_relationship(self) -> ObjectInstanceRelationship: + instances = super( + ExportedRelationshipMeasurementSemantics, self + ).instance_relationship + return ( + instances + if self.relationship.declaration.projects_parent_child_measurements() + else instances.reverse_endpoints() + ) + + def validate_reciprocal_declarations(self, relationships: tuple[Self, ...]) -> None: + """Require an observed reverse declaration to retain producer and pairs.""" + declaration = self.relationship.declaration + reverse = tuple( + other + for other in relationships + if other.source_subject == self.source_subject + and other.target_subject == self.target_subject + and other.relationship.declaration.source_role != declaration.source_role + ) + if reverse and not any( + other.relationship.declaration.producer_module_number + == declaration.producer_module_number + and other.instance_relationship.slice_count + == self.instance_relationship.slice_count + and frozenset( + zip( + other.instance_relationship.source_keys, + other.instance_relationship.target_keys, + strict=True, + ) + ) + == frozenset( + zip( + self.instance_relationship.source_keys, + self.instance_relationship.target_keys, + strict=True, + ) + ) + for other in reverse + ): + raise ValueError( + "Relationship reverse declarations disagree on producer or directed pairs." + ) + + @classmethod + def object_rows( + cls, + object_tables: tuple[RuntimeScopedMeasurementTable, ...], + subject: RuntimeMeasurementSubjectKey, + image_number_offset: RuntimeImageNumberOffset, + ) -> dict[ObjectInstanceKey, RuntimeMeasurementRowMapping]: + rows: dict[ObjectInstanceKey, RuntimeMeasurementRowMapping] = {} + for scoped_table in object_tables: + table = scoped_table.table + if ( + RuntimeMeasurementSubjectKey.from_table_subject(table.subject) + != subject + ): + continue + for values in table.rows.iter_row_mappings(): + row = RuntimeMeasurementRowMapping(measurement_row_mapping(values)) + object_id = row.object_label() + if object_id is None: + raise ValueError( + f"Relationship endpoint {subject.name!r} lacks an object identity." + ) + key = image_number_offset.object_instance_key(row.row, object_id) + if key.slice_index is None: + raise ValueError( + "Relationship object rows require explicit image identities." + ) + if key in rows and rows[key].row != row.row: + raise ValueError( + f"Relationship endpoint {subject.name!r} has conflicting object rows for {key!r}." + ) + rows[key] = row + return rows + + def source_domain( + self, object_label_catalog: RuntimeObjectLabelInstanceCatalog + ) -> tuple[ObjectInstanceKey, ...]: + del object_label_catalog + return tuple( + self.object_rows( + self.object_tables, self.source_subject, self.image_number_offset + ) + ) + + def target_domain( + self, object_label_catalog: RuntimeObjectLabelInstanceCatalog + ) -> tuple[ObjectInstanceKey, ...]: + del object_label_catalog + return tuple( + self.object_rows( + self.object_tables, self.target_subject, self.image_number_offset + ) + ) + + def child_keys_by_parent( + self, object_label_catalog: RuntimeObjectLabelInstanceCatalog + ) -> dict[ObjectInstanceKey, tuple[ObjectInstanceKey, ...]]: + return self.instance_relationship.child_keys_by_parent( + declared_source_keys=self.source_domain(object_label_catalog) + ) + + def validate_recorded_measurements(self) -> None: + """Check all edges and zero-child rows before suppressing redundant facts.""" + parents = self.object_rows( + self.object_tables, self.source_subject, self.image_number_offset + ) + children = self.object_rows( + self.object_tables, self.target_subject, self.image_number_offset + ) + original = self.relationship.payload + for source_id, target_id, slice_index in zip( + original.source_ids, + original.target_ids, + original.slice_indices, + strict=True, + ): + parent_id, child_id = ( + (source_id, target_id) + if self.relationship.declaration.projects_parent_child_measurements() + else (target_id, source_id) + ) + if ( + child_id <= 0 + or ObjectInstanceKey(child_id, slice_index) not in children + ): + raise ValueError( + "Relationship table refers to an absent child endpoint." + ) + if ( + parent_id > 0 + and ObjectInstanceKey(parent_id, slice_index) not in parents + ): + raise ValueError( + "Relationship table refers to an absent parent endpoint." + ) + instances = self.instance_relationship + pairs = tuple(zip(instances.source_keys, instances.target_keys, strict=True)) + if len(set(pairs)) != len(pairs): + raise ValueError("Relationship table contains duplicate directed edges.") + parent_by_child = instances.parent_key_by_child() + if len(parent_by_child) != len(pairs): + raise ValueError( + "Relationship table assigns contradictory parents to one child." + ) + if ( + not set(instances.source_keys) <= parents.keys() + or not set(instances.target_keys) <= children.keys() + ): + raise ValueError("Relationship table refers to an absent object endpoint.") + parent_feature = DirectParentReferenceFeatureDeclaration.feature_name( + DirectParentReferenceMeasurementFeature( + self.relationship.declaration.source.name + if self.relationship.declaration.projects_parent_child_measurements() + else self.relationship.declaration.target.name + ) + ) + for child, row in children.items(): + observed = row.first_value((normalize_runtime_identifier(parent_feature),)) + if observed is None: + raise ValueError( + f"Relationship child {child!r} lacks its explicit parent measurement." + ) + expected = ( + parent_by_child[child].object_id if child in parent_by_child else 0 + ) + if MeasurementScalarLiteral(observed).integer_value != expected: + raise ValueError( + f"Relationship edges disagree with {parent_feature!r} at child {child!r}." + ) + counts = instances.child_keys_by_parent(declared_source_keys=parents) + count_feature = ChildCountFeatureDeclaration.feature_name( + self.relationship.declaration.target.name + if self.relationship.declaration.projects_parent_child_measurements() + else self.relationship.declaration.source.name + ) + for parent, row in parents.items(): + observed = row.first_value((normalize_runtime_identifier(count_feature),)) + if observed is None or MeasurementScalarLiteral( + observed + ).integer_value != len(counts[parent]): + raise ValueError( + f"Relationship edges disagree with {count_feature!r} at parent {parent!r}." + ) + + @classmethod + def correlated_object_relationships( + cls, + object_tables: tuple[RuntimeScopedMeasurementTable, ...], + image_number_offset: RuntimeImageNumberOffset, + ) -> dict[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship]: + """Retain exact native parent references, independently of value counters.""" + pairs: dict[ + RuntimeMeasurementFeatureKey, + list[tuple[ObjectInstanceKey, ObjectInstanceKey]], + ] = {} + for scoped_table in object_tables: + table = scoped_table.table + if table.subject.scope is not MeasurementScope.OBJECT: + continue + child_name = normalize_runtime_identifier(table.subject.name) + for field in table.rows.fields: + identity = DirectParentReferenceFeatureDeclaration.from_feature_name( + field.name + ) + if identity is None: + continue + key = RuntimeMeasurementFeatureKey( + RuntimeMeasurementSubjectKey( + MeasurementScope.RELATIONSHIP, child_name + ), + normalize_runtime_identifier(identity.parent_object_name), + ) + values = pairs.setdefault(key, []) + for raw_row in table.rows.iter_row_mappings(): + row = RuntimeMeasurementRowMapping(measurement_row_mapping(raw_row)) + child_id = row.object_label() + parent_id = MeasurementScalarLiteral( + row.row[field.name] + ).integer_value + if child_id is None or parent_id is None or parent_id < 0: + raise ValueError( + "Relationship measurements require integral endpoint identities." + ) + child_key = image_number_offset.object_instance_key( + row.row, child_id + ) + if child_key.slice_index is None: + raise ValueError( + "Relationship measurements require explicit image identities." + ) + if parent_id > 0: + values.append( + ( + ObjectInstanceKey(parent_id, child_key.slice_index), + child_key, + ) + ) + result = {} + for key, values in pairs.items(): + ordered = tuple( + sorted( + values, + key=lambda pair: ( + pair[0].slice_index, + pair[0].object_id, + pair[1].object_id, + ), + ) + ) + graph = ObjectInstanceRelationship( + tuple(pair[0] for pair in ordered), tuple(pair[1] for pair in ordered) + ) + if len(set(graph.target_keys)) != len(graph.target_keys): + raise ValueError( + "Relationship measurements assign duplicate or contradictory child identities." + ) + parent_subject = RuntimeMeasurementSubjectKey( + MeasurementScope.OBJECT, key.feature_name + ) + parents = cls.object_rows( + object_tables, parent_subject, image_number_offset + ) + if not set(graph.source_keys) <= parents.keys(): + raise ValueError( + "Relationship measurements refer to an absent parent endpoint." + ) + children_by_parent = graph.child_keys_by_parent( + declared_source_keys=parents + ) + count_field = normalize_runtime_identifier( + ChildCountFeatureDeclaration.feature_name(key.subject.name) + ) + for parent, row in parents.items(): + observed = row.first_value((count_field,)) + if observed is not None and MeasurementScalarLiteral( + observed + ).integer_value != len(children_by_parent[parent]): + raise ValueError( + f"Relationship measurements disagree with {count_field!r} at parent {parent!r}." + ) + result[key] = graph + return result diff --git a/openhcs/core/runtime_equivalence.py b/openhcs/core/runtime_equivalence.py index 244f9edde5..f83d3f4251 100644 --- a/openhcs/core/runtime_equivalence.py +++ b/openhcs/core/runtime_equivalence.py @@ -69,6 +69,8 @@ MeasurementTable, ) from openhcs.core.runtime_relationships import ( + ObjectInstanceKey, + ObjectInstanceRelationship, ObjectRelationship, ) from openhcs.core.equivalence.policy import ( @@ -141,6 +143,7 @@ from openhcs.core.equivalence.relationships import ( RelationshipAggregateFeatureSemantics, RelationshipMeasurementSemantics, + ExportedRelationshipMeasurementSemantics, RuntimeObjectRelationshipIdentity, RuntimeScopedMeasurementTable, object_measurement_values_by_label, @@ -1708,7 +1711,7 @@ def project_measurement_fact_counts(self) -> RuntimeMeasurementFactCounterMap: str | None, ] RuntimeCellSignatureCachePayload = tuple[str, str] -RuntimeMeasurementSnapshotCachePayload = tuple[ +RuntimeMeasurementFactCachePayload = tuple[ tuple[ RuntimeMeasurementFeatureCachePayload, tuple[tuple[RuntimeCellSignatureCachePayload, int], ...], @@ -1717,11 +1720,24 @@ def project_measurement_fact_counts(self) -> RuntimeMeasurementFactCounterMap: ] +RuntimeRelationshipCorrelationCachePayload = tuple[ + tuple[RuntimeMeasurementFeatureCachePayload, tuple[tuple[tuple[int, int | None], tuple[int, int | None]], ...], int | None], + ..., +] +RuntimeMeasurementSnapshotCachePayload = tuple[ + RuntimeMeasurementFactCachePayload, + RuntimeRelationshipCorrelationCachePayload | None, +] + + @dataclass(slots=True) class RuntimeMeasurementSnapshot: """Semantic measurement facts independent of table layout.""" measurement_fact_counts: RuntimeMeasurementFactCounterMapping + correlated_relationships: ( + Mapping[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship] | None + ) = None @classmethod def from_output_snapshot( @@ -1736,18 +1752,29 @@ def from_output_snapshot( policy=policy, known_source_names=known_source_names, ) - state.record_measurement_tables( - tuple( - RuntimeScopedMeasurementTable(measurement_table) - for table in snapshot.tables - for measurement_table in table.measurement_tables( - policy.measurement_dialect - ) - ), - None, + tables = tuple( + RuntimeScopedMeasurementTable(measurement_table) + for table in ExportedRelationshipMeasurementSemantics.validated_output_tables( + snapshot.tables, policy + ) + for measurement_table in table.measurement_tables( + policy.measurement_dialect + ) + ) + image_offset = RuntimeImageNumberOffset.from_runtime_rows( + row for table in tables for row in table.table.rows.iter_row_mappings() ) + correlations = ( + ExportedRelationshipMeasurementSemantics.correlated_object_relationships( + tables, image_offset + ) + ) + state.record_measurement_tables(tables, None) state.required_measurement_keys = frozenset(state.explicit_measurement_keys) - return cls(measurement_fact_counts=state.project_measurement_fact_counts()) + return cls( + measurement_fact_counts=state.project_measurement_fact_counts(), + correlated_relationships=correlations, + ) @classmethod def from_artifact_execution_observation( @@ -1779,48 +1806,100 @@ def __post_init__(self) -> None: } ) + if self.correlated_relationships is not None: + self.correlated_relationships = MappingProxyType( + dict(self.correlated_relationships) + ) + @property def is_empty(self) -> bool: return not self.measurement_fact_counts - def to_cache_payload( + def relationship_differences( self, - ) -> RuntimeMeasurementSnapshotCachePayload: - """Return a stable semantic cache payload for repeated equivalence checks.""" - return tuple( + candidate: ( + Mapping[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship] | None + ), + ) -> tuple[RuntimeEquivalenceDifference, ...]: + """Compare saved correlations; None denotes the typed value-only scope.""" + if self.correlated_relationships is None or candidate is None: + return () + if self.correlated_relationships == candidate: + return () + return ( + RuntimeEquivalenceDifference( + RuntimeEquivalenceDifferenceKind.MEASUREMENT_CONTENT, + "directed relationship endpoint/image correlations differ", + ), + ) + + def to_cache_payload(self) -> RuntimeMeasurementSnapshotCachePayload: + """Transport numeric facts and known/unknown relationship evidence together.""" + facts = tuple( ( feature.to_cache_payload(), tuple( (value.to_cache_payload(), int(count)) for value, count in sorted( - values.items(), - key=lambda item: item[0].sort_key, + values.items(), key=lambda item: item[0].sort_key ) ), ) for feature, values in sorted( - self.measurement_fact_counts.items(), - key=lambda item: item[0].sort_key, + self.measurement_fact_counts.items(), key=lambda item: item[0].sort_key ) ) + relationships = None + if self.correlated_relationships is not None: + relationships = tuple( + ( + key.to_cache_payload(), + tuple( + ( + (source.object_id, source.slice_index), + (target.object_id, target.slice_index), + ) + for source, target in zip( + value.source_keys, value.target_keys, strict=True + ) + ), + value.slice_count, + ) + for key, value in sorted( + self.correlated_relationships.items(), + key=lambda item: item[0].sort_key, + ) + ) + return facts, relationships @classmethod def from_cache_payload( - cls, - payload: RuntimeMeasurementSnapshotCachePayload, + cls, payload: RuntimeMeasurementSnapshotCachePayload ) -> "RuntimeMeasurementSnapshot": - """Rebuild a semantic measurement snapshot from cache payload data.""" - measurement_fact_counts: RuntimeMeasurementFactCounterMap = {} - for feature_payload, values_payload in payload: # type: ignore[union-attr] - counter: Counter[RuntimeCellSignature] = Counter() - for value_payload, count in values_payload: - counter[RuntimeCellSignature.from_cache_payload(value_payload)] = int( - count + """Restore the same evidence scope, retaining every directed endpoint pair.""" + fact_payload, relationship_payload = payload + facts = { + RuntimeMeasurementFeatureKey.from_cache_payload(feature): Counter( + { + RuntimeCellSignature.from_cache_payload(value): count + for value, count in values + } + ) + for feature, values in fact_payload + } + relationships = None + if relationship_payload is not None: + relationships = { + RuntimeMeasurementFeatureKey.from_cache_payload( + key + ): ObjectInstanceRelationship( + tuple(ObjectInstanceKey(*source) for source, _target in pairs), + tuple(ObjectInstanceKey(*target) for _source, target in pairs), + slice_count, ) - measurement_fact_counts[ - RuntimeMeasurementFeatureKey.from_cache_payload(feature_payload) - ] = counter - return cls(measurement_fact_counts=measurement_fact_counts) + for key, pairs, slice_count in relationship_payload + } + return cls(facts, relationships) @dataclass(slots=True) @@ -1881,9 +1960,18 @@ def runtime_measurement_equivalence( *, policy: RuntimeEquivalencePolicy = RuntimeEquivalencePolicy(), ) -> RuntimeEquivalenceReport: - """Compare precomputed semantic measurement snapshots.""" + """Compare measurement facts and any jointly supplied saved edge correlations. + + Typed artifact snapshots retain their value-only scope when global saved + image numbering is unavailable. Full saved qualification must use the + known correlation evidence produced by ``from_output_snapshot`` on both + sides; an UNKNOWN typed projection is not such a qualification. + """ return RuntimeEquivalenceReport( - differences=_measurement_differences(reference, candidate, policy) + differences=( + *_measurement_differences(reference, candidate, policy), + *reference.relationship_differences(candidate.correlated_relationships), + ) ) diff --git a/openhcs/core/runtime_relationships.py b/openhcs/core/runtime_relationships.py index 1d542b67e1..ea3b158552 100644 --- a/openhcs/core/runtime_relationships.py +++ b/openhcs/core/runtime_relationships.py @@ -27,7 +27,12 @@ from abc import abstractmethod from collections.abc import Iterable from openhcs.core.registry_strategies import MostDerivedContextStrategyMixin -from openhcs.core.runtime_measurements import MeasurementRowAxisField +from openhcs.core.runtime_measurements import ( + MeasurementRowAxisField, + ObjectReferenceFeatureMarker, + RuntimeMeasurementFeatureDeclaration, + MeasurementScalarLiteral, +) from openhcs.core.source_spatial_domain import SourceSpatialDomainAdapter from openhcs.core.runtime_object_label_domains import ObjectLabelDomainMetadataStrategy from openhcs.core.runtime_object_label_domains import ObjectLabelDomainScope @@ -41,10 +46,106 @@ from openhcs.core.runtime_sparse_labels import SparseIJVLabelRows +class DirectParentReferenceFeatureMarker(ObjectReferenceFeatureMarker): + """Semantic marker for a child's direct parent-object reference.""" + + +@dataclass(frozen=True, slots=True) +class DirectParentReferenceMeasurementFeature: + """Nominal identity encoded by a ``Parent_`` measurement name.""" + + parent_object_name: str + + def __post_init__(self) -> None: + if not isinstance(self.parent_object_name, str) or not self.parent_object_name: + raise ValueError("Direct parent-reference object name cannot be empty.") + + +class DirectParentReferenceFeatureDeclaration(RuntimeMeasurementFeatureDeclaration): + """Parse and render direct parent references at their row-production owner.""" + + declaration_key = "direct_parent_reference" + semantic_marker_types = (DirectParentReferenceFeatureMarker,) + prefix = "Parent_" + + @classmethod + def from_feature_name( + cls, + feature_name: str, + ) -> DirectParentReferenceMeasurementFeature | None: + if not feature_name.startswith(cls.prefix): + return None + parent_object_name = feature_name[len(cls.prefix) :] + if not parent_object_name: + return None + return DirectParentReferenceMeasurementFeature(parent_object_name) + + @classmethod + def feature_name(cls, identity: object) -> str: + if not isinstance(identity, DirectParentReferenceMeasurementFeature): + raise TypeError( + f"{cls.__name__}.feature_name requires " + "DirectParentReferenceMeasurementFeature." + ) + return f"{cls.prefix}{identity.parent_object_name}" + + +class ChildCountFeatureDeclaration(RuntimeMeasurementFeatureDeclaration): + """Own the child-name grammar shared by export readers and CP producers.""" + + declaration_key = "child_count_reference" + prefix = "Children_" + suffix = "_Count" + + @classmethod + def from_feature_name(cls, feature_name: str) -> str | None: + if not feature_name.startswith(cls.prefix) or not feature_name.endswith( + cls.suffix + ): + return None + name = feature_name[len(cls.prefix) : -len(cls.suffix)].strip() + return name or None + + @classmethod + def feature_name(cls, identity: str) -> str: + name = identity.strip() + if not name: + raise ValueError( + "Child-count feature requires a non-empty child object name." + ) + return f"{cls.prefix}{name}{cls.suffix}" + + @dataclass(frozen=True) class ObjectRelationshipDeclaration(ArtifactSpecRelation): """Authoritative directed relationship semantics for one artifact output.""" + exported_field_names: ClassVar[tuple[str, ...]] = ( + "relationship_type", + "source_role", + "target_role", + "source_object", + "target_object", + "producer_module_number", + ) + + def exported_columns(self) -> dict[str, object]: + """Render the declaration owned by every exported edge row.""" + return dict( + zip( + self.exported_field_names, + ( + self.relationship_type, + self.source_role, + self.target_role, + self.source.name, + self.target.name, + self.producer_module_number, + ), + strict=True, + ) + ) + relation_key = "object_relationship_declaration" target_plan_type: ClassVar[type[ArtifactPlan] | None] = None target: ArtifactSpecRef @@ -122,6 +223,45 @@ def parent_child( target_runtime_slice_offset=target_runtime_slice_offset, ) + @classmethod + def from_exported_values( + cls, + relationship_type: str, + source_role: str, + target_role: str, + source_object: str, + target_object: str, + producer_module_number: object, + ) -> Self: + """Restore the complete declared parent/child edge grammar from CSV.""" + source = ArtifactSpec.output(source_object, ObjectLabelsArtifactType).ref() + target = ArtifactSpec.output(target_object, ObjectLabelsArtifactType).ref() + module = MeasurementScalarLiteral(producer_module_number).integer_value + if module is None: + raise ValueError("Relationship producer module number is required.") + canonical = cls.parent_child( + source=source, target=target, producer_module_number=module + ) + roles = (source_role, target_role) + canonical_roles = (canonical.source_role, canonical.target_role) + fields = (canonical.source_id_field, canonical.target_id_field) + if roles == canonical_roles: + source_field, target_field = fields + elif roles == canonical_roles[::-1]: + target_field, source_field = fields + else: + raise ValueError(f"Unsupported exported relationship roles {roles!r}.") + return cls( + source=source, + target=target, + relationship_type=relationship_type, + source_role=roles[0], + target_role=roles[1], + source_id_field=source_field, + target_id_field=target_field, + producer_module_number=module, + ) + def require_target_spec(self, spec: ArtifactSpec) -> None: super().require_target_spec(spec) if not issubclass(spec.artifact_type, ObjectLineageArtifactType): @@ -244,16 +384,9 @@ def project_runtime_slice(self, slice_index: int) -> "ObjectRelationship": def relationship_columns(self) -> dict[str, Any]: """Return semantic relationship columns without payload provenance.""" - columns = { - "relationship_type": self.declaration.relationship_type, - "source_role": self.declaration.source_role, - "target_role": self.declaration.target_role, - "source_object": self.declaration.source.name, - "target_object": self.declaration.target.name, - "producer_module_number": self.declaration.producer_module_number, - self.declaration.source_id_field: self.payload.source_ids, - self.declaration.target_id_field: self.payload.target_ids, - } + columns = self.declaration.exported_columns() + columns[self.declaration.source_id_field] = self.payload.source_ids + columns[self.declaration.target_id_field] = self.payload.target_ids if self.payload.slice_indices: columns["slice_index"] = self.payload.slice_indices if self.payload.slice_count is not None: @@ -951,6 +1084,10 @@ def child_keys_by_parent( ) } + def reverse_endpoints(self) -> Self: + """Transpose the same directed instance pairs without losing their axes.""" + return type(self)(self.target_keys, self.source_keys, self.slice_count) + def parent_key_by_child(self) -> dict[ObjectInstanceKey, ObjectInstanceKey]: """Return source identity for each target identity.""" return dict(zip(self.target_keys, self.source_keys, strict=True)) diff --git a/openhcs/interop/cellprofiler/measurement_lookup.py b/openhcs/interop/cellprofiler/measurement_lookup.py index 8d285990a6..120db857d4 100644 --- a/openhcs/interop/cellprofiler/measurement_lookup.py +++ b/openhcs/interop/cellprofiler/measurement_lookup.py @@ -9,6 +9,7 @@ from metaclass_registry import AutoRegisterMeta +from openhcs.core.runtime_relationships import ChildCountFeatureDeclaration from openhcs.core.public_api import declared_public_names from openhcs.core.measurement_feature_queries import measurement_values_for_feature @@ -151,19 +152,10 @@ class CellProfilerChildCountFeatureParser(CellProfilerMeasurementFeatureParser): kind = CellProfilerMeasurementFeatureKind.CHILD_COUNT kind_key = kind.value - prefix = "Children_" - suffix = "_Count" - def parse_feature( - self, - feature_name: str, - ) -> CellProfilerMeasurementFeature | None: - if not feature_name.startswith(self.prefix): - return None - if not feature_name.endswith(self.suffix): - return None - object_name = feature_name[len(self.prefix) : -len(self.suffix)].strip() - if not object_name: + def parse_feature(self, feature_name: str) -> CellProfilerMeasurementFeature | None: + object_name = ChildCountFeatureDeclaration.from_feature_name(feature_name) + if object_name is None: return None return CellProfilerMeasurementFeature( name=feature_name, @@ -172,18 +164,13 @@ def parse_feature( ) def feature_from_object_name( - self, - object_name: str, + self, object_name: str ) -> CellProfilerMeasurementFeature: - normalized = object_name.strip() - if not normalized: - raise ValueError( - "Child-count feature requires a non-empty child object name." - ) + feature_name = ChildCountFeatureDeclaration.feature_name(object_name) return CellProfilerMeasurementFeature( - name=f"{self.prefix}{normalized}{self.suffix}", + name=feature_name, kind=CellProfilerMeasurementFeatureKind.CHILD_COUNT, - object_name=normalized, + object_name=object_name.strip(), ) diff --git a/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py b/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py index f34f814108..5849251569 100644 --- a/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py +++ b/openhcs/interop/cellprofiler/runtime/relationship_measurement_rows.py @@ -17,8 +17,6 @@ ) from openhcs.core.runtime_measurements import ( MeasurementRowAxisField, - ObjectReferenceFeatureMarker, - RuntimeMeasurementFeatureDeclaration, ) from openhcs.core.runtime_object_label_domains import ( ObjectLabelDomainScope, @@ -35,6 +33,8 @@ from openhcs.core.runtime_relationships import ( ObjectRelationship, ObjectRelationshipDeclaration, + DirectParentReferenceFeatureDeclaration, + DirectParentReferenceMeasurementFeature, ) from openhcs.core.runtime_image_values import ( image_payload_metadata, @@ -53,50 +53,6 @@ ) -class DirectParentReferenceFeatureMarker(ObjectReferenceFeatureMarker): - """Semantic marker for a child's direct parent-object reference.""" - - -@dataclass(frozen=True, slots=True) -class DirectParentReferenceMeasurementFeature: - """Nominal identity encoded by a ``Parent_`` measurement name.""" - - parent_object_name: str - - def __post_init__(self) -> None: - if not isinstance(self.parent_object_name, str) or not self.parent_object_name: - raise ValueError("Direct parent-reference object name cannot be empty.") - - -class DirectParentReferenceFeatureDeclaration(RuntimeMeasurementFeatureDeclaration): - """Parse and render direct parent references at their row-production owner.""" - - declaration_key = "direct_parent_reference" - semantic_marker_types = (DirectParentReferenceFeatureMarker,) - prefix = "Parent_" - - @classmethod - def from_feature_name( - cls, - feature_name: str, - ) -> DirectParentReferenceMeasurementFeature | None: - if not feature_name.startswith(cls.prefix): - return None - parent_object_name = feature_name[len(cls.prefix) :] - if not parent_object_name: - return None - return DirectParentReferenceMeasurementFeature(parent_object_name) - - @classmethod - def feature_name(cls, identity: object) -> str: - if not isinstance(identity, DirectParentReferenceMeasurementFeature): - raise TypeError( - f"{cls.__name__}.feature_name requires " - "DirectParentReferenceMeasurementFeature." - ) - return f"{cls.prefix}{identity.parent_object_name}" - - @dataclass(frozen=True, slots=True) class RelationshipMeasurementRows: """Project parent-child relationship payloads into CP object measurement rows.""" diff --git a/openhcs/processing/backends/cellprofiler/relationships.py b/openhcs/processing/backends/cellprofiler/relationships.py index b69c31d2f6..0f6f2d0380 100644 --- a/openhcs/processing/backends/cellprofiler/relationships.py +++ b/openhcs/processing/backends/cellprofiler/relationships.py @@ -5,6 +5,9 @@ from dataclasses import dataclass, replace from enum import Enum from typing import TYPE_CHECKING, Annotated +from openhcs.core.runtime_relationships import ( + DirectParentReferenceFeatureMarker, +) from openhcs.core.alias_property import AliasProperty from openhcs.core.artifacts import ( ArtifactInputPlan, @@ -71,7 +74,6 @@ ObjectLocationMeasurementRows, ) from openhcs.interop.cellprofiler.runtime.relationship_measurement_rows import ( - DirectParentReferenceFeatureMarker, RelationshipMeasurementRows, ) diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index 90f612d3e4..43240f69c8 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -10,6 +10,10 @@ import openhcs.processing.backends.cellprofiler.secondary as iso import openhcs.processing.backends.cellprofiler.secondary as ito +from openhcs.core.runtime_relationships import ( + DirectParentReferenceFeatureDeclaration, + DirectParentReferenceMeasurementFeature, +) from openhcs.constants.constants import ( AllComponents, Backend, @@ -252,8 +256,6 @@ CellProfilerOutputRecorder, ) from openhcs.interop.cellprofiler.runtime.relationship_measurement_rows import ( - DirectParentReferenceFeatureDeclaration, - DirectParentReferenceMeasurementFeature, RelationshipMeasurementRows, ) from openhcs.processing.backends.cellprofiler.alignment import ( diff --git a/tests/unit/test_cellprofiler_relationship_contracts.py b/tests/unit/test_cellprofiler_relationship_contracts.py index 74ee1d575a..64a352ebb2 100644 --- a/tests/unit/test_cellprofiler_relationship_contracts.py +++ b/tests/unit/test_cellprofiler_relationship_contracts.py @@ -6,6 +6,10 @@ import numpy as np +from openhcs.core.runtime_relationships import ( + DirectParentReferenceFeatureDeclaration, + DirectParentReferenceMeasurementFeature, +) from openhcs.core.artifacts import ( ArtifactInputPlan, ArtifactOutputPlan, @@ -49,10 +53,6 @@ ) from openhcs.interop.cellprofiler.pipeline_import import import_cellprofiler_pipeline from openhcs.interop.cellprofiler.setting_names import setting_values -from openhcs.interop.cellprofiler.runtime.relationship_measurement_rows import ( - DirectParentReferenceFeatureDeclaration, - DirectParentReferenceMeasurementFeature, -) from openhcs.processing.backends.cellprofiler.neighbors import ( MeasureObjectNeighborsModule, ) diff --git a/tests/unit/test_exported_relationship_equivalence.py b/tests/unit/test_exported_relationship_equivalence.py new file mode 100644 index 0000000000..60273f2fac --- /dev/null +++ b/tests/unit/test_exported_relationship_equivalence.py @@ -0,0 +1,393 @@ +"""Saved directed edges must retain the native object-row correlations.""" + +from pathlib import Path + +import pytest + +from openhcs.core.equivalence.outputs import RuntimeOutputSnapshot +from openhcs.core.runtime_equivalence import ( + RuntimeMeasurementSnapshot, + runtime_measurement_equivalence, +) + +_HEADER = ( + "relationship_type,source_role,target_role,source_object,target_object," + "producer_module_number,parent_id,child_id,image_number,slice_count\n" +) + + +def _saved_outputs( + root: Path, *, edges: bool, swapped: bool = False +) -> RuntimeOutputSnapshot: + root.mkdir() + (root / "Cells.csv").write_text( + "ImageNumber,ObjectNumber,Children_Nuclei_Count\n" + "1,1,1\n1,2,1\n1,3,0\n2,1,1\n2,2,1\n2,3,0\n" + ) + parents = (2, 1) if swapped else (1, 2) + (root / "Nuclei.csv").write_text( + "ImageNumber,ObjectNumber,Parent_Cells\n" + + "".join( + f"{image},{child},{parents[child - 1]}\n" + for image in (1, 2) + for child in (1, 2) + ) + ) + if edges: + (root / "Relationships.csv").write_text( + _HEADER + + "".join( + f"{kind},{source_role},{target_role},{source},{target},11," + f"{parents[child - 1]},{child},{image},2\n" + for kind, source_role, target_role, source, target in ( + ("Parent", "parent", "child", "Cells", "Nuclei"), + ("Child", "child", "parent", "Nuclei", "Cells"), + ) + for image in (1, 2) + for child in (1, 2) + ) + ) + return RuntimeOutputSnapshot.from_output_root(root) + + +def test_saved_relationship_edges_join_native_parent_and_zero_child_counts( + tmp_path: Path, +) -> None: + reference = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "native", edges=False) + ) + candidate = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "openhcs", edges=True) + ) + assert runtime_measurement_equivalence(reference, candidate).is_equivalent + + +def test_relationship_endpoint_permutation_cannot_hide_in_equal_value_histograms( + tmp_path: Path, +) -> None: + reference = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "native", edges=False) + ) + candidate = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "openhcs", edges=True, swapped=True) + ) + assert not runtime_measurement_equivalence(reference, candidate).is_equivalent + + +def test_relationship_rows_must_agree_with_explicit_parent_measurements( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "openhcs", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.rows = tuple( + tuple( + "2" if index == 6 and row[7] == "1" and row[8] == "1" else value + for index, value in enumerate(row) + ) + for row in table.rows + ) + with pytest.raises(ValueError, match="relationship|Relationship"): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_ordinary_image_row_conflicts_remain_rejected(tmp_path: Path) -> None: + root = tmp_path / "ordinary" + root.mkdir() + (root / "Image.csv").write_text("ImageNumber,Count_Nuclei\n1,2\n1,3\n") + with pytest.raises(ValueError, match="conflicting observed values"): + RuntimeMeasurementSnapshot.from_output_snapshot( + RuntimeOutputSnapshot.from_output_root(root) + ) + + +@pytest.mark.parametrize( + "mutation, message", + [ + ( + lambda header, rows: (header, rows[:-1]), + "Relationship edges disagree|reverse declarations", + ), + (lambda header, rows: (header, (*rows, rows[0])), "duplicate directed edges"), + ( + lambda header, rows: ( + header, + tuple( + tuple( + "12" if i == 5 and row[0] == "Child" else value + for i, value in enumerate(row) + ) + for row in rows + ), + ), + "reverse declarations", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple( + "neighbor_source" if i == 1 else value + for i, value in enumerate(row) + ) + for row in rows + ), + ), + "Unsupported exported relationship roles", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple("999" if i == 7 else value for i, value in enumerate(row)) + for row in rows + ), + ), + "absent child", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple("2.5" if i == 6 else value for i, value in enumerate(row)) + for row in rows + ), + ), + "nonnegative integers", + ), + ( + lambda header, rows: ( + header, + tuple( + tuple("2.5" if i == 9 else value for i, value in enumerate(row)) + for row in rows + ), + ), + "slice count must be an integer", + ), + ( + lambda header, rows: (header[1:], tuple(row[1:] for row in rows)), + "incomplete declaration", + ), + (lambda header, rows: (header, ()), "no determining declaration"), + ], +) +def test_saved_edge_schema_and_coverage_rejects_malformed_inputs( + tmp_path: Path, mutation, message: str +) -> None: + snapshot = _saved_outputs(tmp_path / "openhcs", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.header, table.rows = mutation(table.header, table.rows) + with pytest.raises((ValueError, TypeError), match=message): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_saved_relationship_counts_include_noncontiguous_zero_child_parents( + tmp_path: Path, +) -> None: + native = _saved_outputs(tmp_path / "native", edges=False) + candidate = _saved_outputs(tmp_path / "openhcs", edges=True) + for snapshot in (native, candidate): + for table in snapshot.tables: + table.rows = tuple( + tuple( + ( + "30" + if table.path.stem == "Cells" and i == 1 and value == "3" + else value + ) + for i, value in enumerate(row) + ) + for row in table.rows + ) + a = RuntimeMeasurementSnapshot.from_output_snapshot(native) + b = RuntimeMeasurementSnapshot.from_output_snapshot(candidate) + assert ( + a.correlated_relationships is not None + and b.correlated_relationships is not None + ) + assert runtime_measurement_equivalence(a, b).is_equivalent + + +def test_snapshot_cache_retains_known_correlations_and_unknown_typed_scope( + tmp_path: Path, +) -> None: + known = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "saved", edges=True) + ) + unknown = RuntimeMeasurementSnapshot(known.measurement_fact_counts) + for snapshot in (known, unknown): + restored = RuntimeMeasurementSnapshot.from_cache_payload( + snapshot.to_cache_payload() + ) + assert restored.measurement_fact_counts == snapshot.measurement_fact_counts + assert restored.correlated_relationships == snapshot.correlated_relationships + assert unknown.correlated_relationships is None + assert known.correlated_relationships is not None + + +def test_cached_relationship_preserves_declared_slice_count_and_endpoint_axes() -> None: + from openhcs.core.equivalence.keys import ( + RuntimeMeasurementFeatureKey, + RuntimeMeasurementSubjectKey, + ) + from openhcs.core.runtime_measurements import MeasurementScope + from openhcs.core.runtime_relationships import ( + ObjectInstanceKey, + ObjectInstanceRelationship, + ) + + key = RuntimeMeasurementFeatureKey( + RuntimeMeasurementSubjectKey(MeasurementScope.RELATIONSHIP, "child"), "parent" + ) + graph = ObjectInstanceRelationship( + (ObjectInstanceKey(2, 0),), (ObjectInstanceKey(5, 1),), 2 + ) + snapshot = RuntimeMeasurementSnapshot({}, {key: graph}) + assert ( + RuntimeMeasurementSnapshot.from_cache_payload( + snapshot.to_cache_payload() + ).correlated_relationships + == snapshot.correlated_relationships + ) + + +def test_saved_relationship_omitted_slice_count_preserves_optional_payload_contract( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "saved", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.header = table.header[:-1] + table.rows = tuple(row[:-1] for row in table.rows) + result = RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + assert result.correlated_relationships is not None + + +def test_saved_relationship_row_order_does_not_change_directed_correlation( + tmp_path: Path, +) -> None: + native = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "native", edges=False) + ) + snapshot = _saved_outputs(tmp_path / "saved", edges=True) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.rows = table.rows[:4] + table.rows[4:][::-1] + result = RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + assert runtime_measurement_equivalence(native, result).is_equivalent + + +def test_unparented_zero_edges_still_validate_actual_child_domain( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "saved", edges=True) + for table in snapshot.tables: + if table.path.stem == "Cells": + table.rows = tuple((*row[:2], "0") for row in table.rows) + elif table.path.stem == "Nuclei": + table.rows = tuple((*row[:2], "0") for row in table.rows) + else: + table.rows = tuple((*row[:6], "0", *row[7:]) for row in table.rows) + result = RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + assert result.correlated_relationships is not None + assert all( + not graph.source_keys for graph in result.correlated_relationships.values() + ) + table = next( + table for table in snapshot.tables if table.path.stem == "Relationships" + ) + table.rows = tuple((*row[:7], "999", *row[8:]) for row in table.rows) + with pytest.raises(ValueError, match="absent child"): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +@pytest.mark.parametrize( + "feature, identity", + [ + ("Children_ Nuclei _Count", "Nuclei"), + ("Children__Count", None), + ("Children_Nuclei", None), + (" children_Nuclei_Count", None), + ], +) +def test_child_count_parser_and_core_declaration_share_original_grammar( + feature: str, identity: str | None +) -> None: + from openhcs.core.runtime_relationships import ChildCountFeatureDeclaration + from openhcs.interop.cellprofiler.measurement_lookup import ( + CellProfilerChildCountFeatureParser, + ) + + assert ChildCountFeatureDeclaration.from_feature_name(feature) == identity + parsed = CellProfilerChildCountFeatureParser().parse_feature(feature) + assert (None if parsed is None else parsed.object_name) == identity + + +@pytest.mark.parametrize("swapped", (False, True)) +def test_physical_inventory_consumes_redundant_edges_and_compares_exact_correlations( + tmp_path: Path, swapped: bool +) -> None: + from benchmark.matched_cellprofiler_batch import _require_compared_output_inventory + from openhcs.core.runtime_exports import RuntimeExportObservation + + reference = _saved_outputs(tmp_path / "native", edges=False) + candidate = _saved_outputs(tmp_path / "saved", edges=True, swapped=swapped) + reference_exports = RuntimeExportObservation.from_output_root(tmp_path / "native") + candidate_exports = RuntimeExportObservation.from_output_root(tmp_path / "saved") + kwargs = dict( + reference_files=frozenset((tmp_path / "native").iterdir()), + candidate_files=frozenset((tmp_path / "saved").iterdir()), + reference_exports=reference_exports, + candidate_exports=candidate_exports, + reference_snapshot=reference, + candidate_snapshot=candidate, + ) + if swapped: + with pytest.raises(RuntimeError, match="correlations differ"): + _require_compared_output_inventory(**kwargs) + else: + _require_compared_output_inventory(**kwargs) + + +@pytest.mark.parametrize( + "parent_value, message", + [("999", "absent parent endpoint"), ("2.5", "integral endpoint")], +) +def test_native_parent_rows_require_real_integral_parent_domain( + tmp_path: Path, parent_value: str, message: str +) -> None: + snapshot = _saved_outputs(tmp_path / "native", edges=False) + table = next(table for table in snapshot.tables if table.path.stem == "Nuclei") + table.rows = tuple((*row[:2], parent_value) for row in table.rows) + with pytest.raises(ValueError, match=message): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_native_child_count_permutation_cannot_hide_in_equal_histograms( + tmp_path: Path, +) -> None: + snapshot = _saved_outputs(tmp_path / "native", edges=False) + table = next(table for table in snapshot.tables if table.path.stem == "Cells") + table.rows = tuple((*row[:2], "0" if row[1] == "1" else "1") for row in table.rows) + with pytest.raises(ValueError, match="disagree with.*children_nuclei_count"): + RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) + + +def test_snapshot_json_cache_retains_correlation_scope(tmp_path: Path) -> None: + import json + + known = RuntimeMeasurementSnapshot.from_output_snapshot( + _saved_outputs(tmp_path / "saved", edges=True) + ) + for snapshot in (known, RuntimeMeasurementSnapshot(known.measurement_fact_counts)): + restored = RuntimeMeasurementSnapshot.from_cache_payload( + json.loads(json.dumps(snapshot.to_cache_payload())) + ) + assert restored.correlated_relationships == snapshot.correlated_relationships + assert restored.measurement_fact_counts == snapshot.measurement_fact_counts From acb26de631be9d451f6421e23e31ace28da5e232 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:34:02 -0400 Subject: [PATCH 090/125] Render authored CPA workspace panels and compare saved semantics (refs #480) --- benchmark/cellprofiler_export_equivalence.py | 33 ++ benchmark/matched_cellprofiler_batch.py | 2 +- docs/development/cpa_workspace_export_480.md | 13 + .../interop/cellprofiler/analyst_export.py | 4 + .../interop/cellprofiler/workspace_export.py | 359 ++++++++++++++++++ .../cellprofiler/export_to_database.py | 89 ++--- .../quality_control_4281.workspace | 71 ++++ .../unit/test_cellprofiler_analyst_export.py | 33 +- .../test_cellprofiler_export_to_database.py | 1 - .../test_cellprofiler_workspace_export.py | 239 ++++++++++++ 10 files changed, 777 insertions(+), 67 deletions(-) create mode 100644 docs/development/cpa_workspace_export_480.md create mode 100644 openhcs/interop/cellprofiler/workspace_export.py create mode 100644 tests/fixtures/cellprofiler/quality_control_4281.workspace create mode 100644 tests/unit/test_cellprofiler_workspace_export.py diff --git a/benchmark/cellprofiler_export_equivalence.py b/benchmark/cellprofiler_export_equivalence.py index ded38131a3..a07723be6e 100644 --- a/benchmark/cellprofiler_export_equivalence.py +++ b/benchmark/cellprofiler_export_equivalence.py @@ -70,6 +70,10 @@ def cellprofiler_database_export_equivalence( _declared_sqlite_table_subjects(candidate_properties), policy, ), + *_workspace_export_differences( + tuple(sorted(reference_root.rglob("*.workspace"))), + _outputs_with_suffix(candidate_exports, ".workspace"), + ), *_properties_export_differences( reference_properties, candidate_properties, @@ -157,6 +161,12 @@ def cellprofiler_native_shard_equivalence( differences.extend( _properties_export_differences(reference_properties, shard_properties) ) + differences.extend( + _workspace_export_differences( + tuple(sorted(reference_root.rglob("*.workspace"))), + tuple(sorted(root.rglob("*.workspace"))), + ) + ) shard_subjects = _declared_sqlite_table_subjects(shard_properties) if shard_subjects != reference_subjects: differences.append( @@ -686,3 +696,26 @@ def _unique_paths_by_name( ) by_name[path.name] = path return by_name + + +def _workspace_export_differences(reference_paths, candidate_paths): + from openhcs.interop.cellprofiler.workspace_export import CPAWorkspacePanel + + differences, reference_by_name, candidate_by_name = _named_output_differences( + reference_paths, candidate_paths, output_label="CPA workspace" + ) + for name in reference_by_name.keys() & candidate_by_name.keys(): + reference = CPAWorkspacePanel.parse_workspace( + reference_by_name[name].read_text() + ) + candidate = CPAWorkspacePanel.parse_workspace( + candidate_by_name[name].read_text() + ) + if reference != candidate: + differences.append( + RuntimeEquivalenceDifference( + RuntimeEquivalenceDifferenceKind.TABLE_CONTENT, + f"CPA workspace {name!r} panel declarations differ.", + ) + ) + return tuple(differences) diff --git a/benchmark/matched_cellprofiler_batch.py b/benchmark/matched_cellprofiler_batch.py index 53441d6777..ff7652e785 100644 --- a/benchmark/matched_cellprofiler_batch.py +++ b/benchmark/matched_cellprofiler_batch.py @@ -196,7 +196,7 @@ def _require_compared_output_inventory( | frozenset( path for path in exports.output_files - if path.suffix.lower() in {".db", ".properties"} + if path.suffix.lower() in {".db", ".properties", ".workspace"} ) ) if files != compared_files: diff --git a/docs/development/cpa_workspace_export_480.md b/docs/development/cpa_workspace_export_480.md new file mode 100644 index 0000000000..44e67d55fd --- /dev/null +++ b/docs/development/cpa_workspace_export_480.md @@ -0,0 +1,13 @@ +# Authored CPA workspace export (#480) + +ExportToDatabase accepted and validated workspace panel settings, then discarded them. The public export wrote SQLite and CPA properties while omitting the requested workspace. Both fresh native QC repetitions retained twelve workspace panels; the original complete-output comparison stayed red. + +The existing export settings now retain the requested workspace and original ordered panel declarations. `CPAWorkspacePanel` owns the external panel grammar, repeated setting binding and authoring transport. Its Scatter/Density leaves share two-axis rendering; Histogram/Box/PlateViewer use the single-axis grammar, with PlateViewer's native `measurement` field. `CPAWorkspaceAxis` derives its Image/Object/Index implementations from their declarations. Image/object measurements use the existing `MeasurementSubject` and database dialect; there is no parallel tool enum or handwritten dispatch roster. + +The public exporter merges workspace text into the existing file bundle and retains its duplicate-file rejection. The shared database-export comparison parses ordered workspace semantics, rejects invalid layout or unknown fields/tools, and reports missing or changed panels. The strict matched inventory now admits `.workspace` because it has an actual comparison consumer. Database and properties rendering remain unchanged. + +Native 4.2.8.1 emits raw subject/feature names in workspace axes, and Object axes always name `Per_Object`, even when separate object tables are selected. These native quirks are preserved, not silently translated to different scientific columns. The near-term QC fixture has only Image/Index axes: both retained native workspace texts match exactly, and all 22 axis references resolve to actual native SQLite columns. This does not establish arbitrary native object-workspace references or shortened column names are valid. The reader validates workflow format version 1 and the numeric producer-version header; the CP producer version is descriptive rather than compared panel meaning. + +The retained native fixture comes from source `6e69b85122d3c1a44857835ba9b3fe77e743240d`, exact matched ten physical images/two image sets and native CP 4.2.8.1. Its SHA256 is `884f14365ed643c8dd7050ea77e416a1b7253dca83ff11d2e01aa54a8e07bdc6`. Source-backed installed native renderer: `cellprofiler/modules/exporttodatabase.py:4799`. The original native inventory red and interrupted test shard remain in `/var/tmp`; the current production pipeline still needs a fresh complete-output gate before #480 closes. + +Ownership evidence reuses the complete NRA class census from the shared derivation audit and the finite original CPA class inventory at `/var/tmp/openhcs-qc-workspace-owner-inventory-20261002.json`. The bounded migration removes the old validate-and-discard workspace algorithm from `ExportToDatabaseModule`; authored setting names remain authoritative there, and the panel owner consumes those exact declarations. These are feature and boundary-format changes authored as exact patches; no automatic codemod or formal scientific equivalence is claimed. diff --git a/openhcs/interop/cellprofiler/analyst_export.py b/openhcs/interop/cellprofiler/analyst_export.py index 8aeacc0359..9ceba032f7 100644 --- a/openhcs/interop/cellprofiler/analyst_export.py +++ b/openhcs/interop/cellprofiler/analyst_export.py @@ -6,6 +6,8 @@ from __future__ import annotations +from openhcs.interop.cellprofiler.workspace_export import CPAWorkspacePanel + from base64 import b64encode from collections import defaultdict from collections.abc import Mapping, Sequence @@ -283,6 +285,8 @@ class CellProfilerDatabaseExportSettings: selected_objects: tuple[str, ...] | None wants_properties_file: bool wants_relationship_tables: bool + wants_workspace_file: bool = False + workspace_panels: tuple[CPAWorkspacePanel, ...] = () maximum_column_name_length: int = 64 location_object: str | None = None plate_type: str | None = None diff --git a/openhcs/interop/cellprofiler/workspace_export.py b/openhcs/interop/cellprofiler/workspace_export.py new file mode 100644 index 0000000000..c32a9a4670 --- /dev/null +++ b/openhcs/interop/cellprofiler/workspace_export.py @@ -0,0 +1,359 @@ +"""Authored CellProfiler Analyst workspace panels and their external grammar.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar, TYPE_CHECKING + +from metaclass_registry import AutoRegisterMeta + +from openhcs.core.runtime_measurements import MeasurementSubject, MeasurementScope +from openhcs.core.runtime_tabular_values import FieldSpec + +from openhcs.interop.cellprofiler.database_column_dialect import ( + CellProfilerDatabaseColumnDialect, +) + +from openhcs.interop.cellprofiler.parser import ModuleBlock, ModuleSetting +from openhcs.interop.cellprofiler.cellprofiler_literals import ( + cellprofiler_setting_literal, +) +from openhcs.interop.cellprofiler.setting_names import optional_setting_value +from openhcs.interop.cellprofiler.settings_binder import parse_cellprofiler_bool + +if TYPE_CHECKING: + from openhcs.interop.cellprofiler.analyst_export import ( + CellProfilerDatabaseExportSettings, + ) + from openhcs.processing.backends.cellprofiler.export_to_database import ( + ExportToDatabaseModule, + ) + + +@dataclass(frozen=True) +class CPAWorkspaceAxis(ABC, metaclass=AutoRegisterMeta): + """One authored axis; irreducible Image/Object/Index projection on leaves.""" + + __registry_key__ = "measurement_type" + __skip_if_no_key__ = True + measurement_type: ClassVar[str | None] = None + object_name: str + measurement: str + index: str + + @classmethod + def from_settings( + cls, kind: str, object_name: str, measurement: str, index: str + ) -> CPAWorkspaceAxis: + axis_type = cls.__registry__.get(kind) + if axis_type is None: + raise ValueError(f"Unsupported CPA workspace measurement type {kind!r}.") + return axis_type(object_name, measurement, index) + + def projected(self, dialect: CellProfilerDatabaseColumnDialect) -> tuple[str, str]: + field, table = self.field_name(dialect), self.table_name(dialect) + if not field or any(c in field for c in "\n\r\t"): + raise ValueError("CPA workspace axis must have a single nonempty field.") + return field, table + + @abstractmethod + def field_name(self, dialect: CellProfilerDatabaseColumnDialect) -> str: + """Project the authored field through the existing measurement dialect.""" + + @abstractmethod + def table_name(self, dialect: CellProfilerDatabaseColumnDialect) -> str: + """Return the original native table role for this axis.""" + + +class CPAImageWorkspaceAxis(CPAWorkspaceAxis): + measurement_type = "Image" + + def field_name(self, dialect): + return dialect.measurement_field( + MeasurementSubject(MeasurementScope.IMAGE, "Image"), + FieldSpec(self.measurement, None), + ).name + + def table_name(self, dialect): + return dialect.image_table_name() + + +class CPAObjectWorkspaceAxis(CPAWorkspaceAxis): + measurement_type = "Object" + + def field_name(self, dialect): + return dialect.measurement_field( + MeasurementSubject(MeasurementScope.OBJECT, self.object_name), + FieldSpec(self.measurement, None), + ).name + + def table_name(self, dialect): + # CP4281 workspace references Per_Object even for per-object exports. + return dialect.combined_object_table_name() + + +class CPAIndexWorkspaceAxis(CPAWorkspaceAxis): + measurement_type = "Index" + + def field_name(self, dialect): + if self.index not in ("ImageNumber", "Group_Index"): + raise ValueError(f"Unsupported CPA workspace index {self.index!r}.") + return self.index + + def table_name(self, dialect): + return dialect.image_table_name() + + +@dataclass(frozen=True) +class CPAWorkspacePanel(ABC, metaclass=AutoRegisterMeta): + """One substitutable workspace tool; shared ordered external rendering.""" + + __registry_key__ = "setting_name" + __skip_if_no_key__ = True + setting_name: ClassVar[str | None] = None + display_name: ClassVar[str] + x_axis_label: ClassVar[str] = "x-axis" + x_table_label: ClassVar[str] = "table" + x: CPAWorkspaceAxis + y: CPAWorkspaceAxis + + @classmethod + def bound_settings( + cls, module: ModuleBlock, declaration: type[ExportToDatabaseModule] + ) -> dict[str, object]: + count_value = optional_setting_value( + module, declaration.workspace_measurement_count_setting + ) + count = 0 if count_value is None else int(count_value) + names = ( + declaration.workspace_display_tool_setting, + declaration.workspace_x_type_setting, + declaration.workspace_x_measurement_setting, + declaration.workspace_x_index_setting, + declaration.workspace_y_type_setting, + declaration.workspace_y_measurement_setting, + declaration.workspace_y_index_setting, + ) + values = tuple(module.get_setting_values(name.canonical) for name in names) + for name, rows in zip(names, values, strict=True): + declaration._require_record_count(count, name, rows) + objects = module.get_setting_values( + declaration.workspace_object_name_setting.canonical + ) + declaration._require_record_count( + count * 2, declaration.workspace_object_name_setting, objects + ) + wants_value = optional_setting_value( + module, declaration.wants_workspace_file_setting + ) + wants = wants_value is not None and parse_cellprofiler_bool(wants_value) + ( + tools, + x_types, + x_measurements, + x_indices, + y_types, + y_measurements, + y_indices, + ) = values + panels = ( + tuple( + cls.from_settings( + tool, + CPAWorkspaceAxis.from_settings( + x_types[i], objects[2 * i], x_measurements[i], x_indices[i] + ), + CPAWorkspaceAxis.from_settings( + y_types[i], objects[2 * i + 1], y_measurements[i], y_indices[i] + ), + ) + for i, tool in enumerate(tools) + ) + if wants + else () + ) + return ( + {"wants_workspace_file": True, "workspace_panels": panels} if wants else {} + ) + + @classmethod + def setting_records( + cls, + declaration: type[ExportToDatabaseModule], + wants: bool, + panels: tuple[CPAWorkspacePanel, ...], + ) -> tuple[ModuleSetting, ...]: + records = [ + ModuleSetting( + declaration.workspace_measurement_count_setting.canonical, + str(len(panels)), + ), + ModuleSetting( + declaration.wants_workspace_file_setting.canonical, + cellprofiler_setting_literal(wants), + ), + ] + for panel in panels: + values = ( + (declaration.workspace_display_tool_setting, panel.setting_name), + (declaration.workspace_x_type_setting, panel.x.measurement_type), + (declaration.workspace_object_name_setting, panel.x.object_name), + (declaration.workspace_x_measurement_setting, panel.x.measurement), + (declaration.workspace_x_index_setting, panel.x.index), + (declaration.workspace_y_type_setting, panel.y.measurement_type), + (declaration.workspace_object_name_setting, panel.y.object_name), + (declaration.workspace_y_measurement_setting, panel.y.measurement), + (declaration.workspace_y_index_setting, panel.y.index), + ) + records.extend( + ModuleSetting(name.canonical, value) for name, value in values + ) + return tuple(records) + + @classmethod + def from_settings( + cls, tool: str, x: CPAWorkspaceAxis, y: CPAWorkspaceAxis + ) -> CPAWorkspacePanel: + panel_type = cls.__registry__.get(tool) + if panel_type is None: + raise ValueError(f"Unsupported CPA workspace display tool {tool!r}.") + panel = panel_type(x, y) + panel.rows(CellProfilerDatabaseColumnDialect()) + return panel + + def rows( + self, dialect: CellProfilerDatabaseColumnDialect + ) -> tuple[tuple[str, str], ...]: + field, table = self.x.projected(dialect) + return ( + (self.x_axis_label, field), + (self.x_table_label, table), + *self.additional_rows(dialect), + ) + + @abstractmethod + def additional_rows( + self, dialect: CellProfilerDatabaseColumnDialect + ) -> tuple[tuple[str, str], ...]: + """Irreducible tool axes beyond the common X declaration.""" + + @classmethod + def parse_workspace( + cls, text: str + ) -> tuple[tuple[str, tuple[tuple[str, str], ...]], ...]: + """Read ordered panel meaning; reject unknown tools, fields and layout.""" + lines = text.splitlines() + if len(lines) < 3 or lines[:2] != [ + "CellProfiler Analyst workflow", + "version: 1", + ]: + raise ValueError("Invalid CPA workspace header.") + if not lines[2].startswith("CP version : ") or not lines[2][13:].isdigit(): + raise ValueError("Invalid CPA workspace CellProfiler version.") + tools = {leaf.display_name: leaf for leaf in cls.__registry__.values()} + panels = [] + current = None + rows = [] + for line in (*lines[3:], ""): + if not line: + if current is not None: + expected = current.external_fields() + if tuple(key for key, value in rows) != expected or any( + not value for key, value in rows + ): + raise ValueError( + f"Invalid CPA workspace {current.display_name} fields." + ) + panels.append((current.display_name, tuple(rows))) + current, rows = None, [] + continue + if current is None: + if line not in tools: + raise ValueError(f"Unknown CPA workspace tool {line!r}.") + current = tools[line] + else: + if not line.startswith("\t") or ": " not in line: + raise ValueError("Invalid CPA workspace panel field layout.") + key, value = line[1:].split(": ", 1) + rows.append((key, value)) + return tuple(panels) + + @classmethod + def external_fields(cls) -> tuple[str, ...]: + return (cls.x_axis_label, cls.x_table_label) + + +class CPASingleAxisWorkspacePanel(CPAWorkspacePanel, ABC): + def additional_rows(self, dialect): + return () + + +class CPATwoAxisWorkspacePanel(CPAWorkspacePanel, ABC): + x_table_label = "x-table" + + def additional_rows(self, dialect): + field, table = self.y.projected(dialect) + return (("y-axis", field), ("y-table", table)) + + @classmethod + def external_fields(cls): + return (*super().external_fields(), "y-axis", "y-table") + + +class CPAScatterWorkspacePanel(CPATwoAxisWorkspacePanel): + setting_name = "ScatterPlot" + display_name = "Scatter" + + +class CPADensityWorkspacePanel(CPATwoAxisWorkspacePanel): + setting_name = "DensityPlot" + display_name = "Density" + + +class CPAHistogramWorkspacePanel(CPASingleAxisWorkspacePanel): + setting_name = "Histogram" + display_name = "Histogram" + + +class CPAPlateViewerWorkspacePanel(CPASingleAxisWorkspacePanel): + setting_name = "PlateViewer" + display_name = "PlateViewer" + x_axis_label = "measurement" + + +class CPABoxWorkspacePanel(CPASingleAxisWorkspacePanel): + setting_name = "BoxPlot" + display_name = "BoxPlot" + + +@dataclass(frozen=True, slots=True) +class CPAWorkspaceRenderer: + """Render the requested workspace through the export's actual table dialect.""" + + dialect: CellProfilerDatabaseColumnDialect + + def render(self, settings: CellProfilerDatabaseExportSettings) -> dict[str, str]: + if not settings.wants_workspace_file: + return {} + if self.dialect.table_prefix != settings.table_prefix: + raise ValueError( + "CPA workspace dialect disagrees with export table prefix." + ) + name = str(Path(settings.sqlite_file).with_suffix("")) + prefix = settings.table_prefix.removesuffix("_") + if prefix: + name = f"{name}_{prefix}" + sections = tuple( + "\n" + + panel.display_name + + "\n" + + "\n".join(f"\t{key}: {value}" for key, value in panel.rows(self.dialect)) + + "\n" + for panel in settings.workspace_panels + ) + return { + f"{name}.workspace": "CellProfiler Analyst workflow\nversion: 1\nCP version : 4281\n" + + "".join(sections) + } diff --git a/openhcs/processing/backends/cellprofiler/export_to_database.py b/openhcs/processing/backends/cellprofiler/export_to_database.py index 2234aaa77f..9a650ab9c9 100644 --- a/openhcs/processing/backends/cellprofiler/export_to_database.py +++ b/openhcs/processing/backends/cellprofiler/export_to_database.py @@ -32,6 +32,10 @@ CellProfilerDatabaseExportSettings, CellProfilerObjectTableMode, ) +from openhcs.interop.cellprofiler.workspace_export import ( + CPAWorkspacePanel, + CPAWorkspaceRenderer, +) from openhcs.interop.cellprofiler.cellprofiler_literals import ( cellprofiler_setting_literal, ) @@ -404,7 +408,7 @@ def postprocess_bound_settings( overwrite_value = optional_setting_value(module, cls.overwrite_mode_setting) if overwrite_value is not None: _parse_overwrite_mode(overwrite_value) - cls._validate_workspace_settings(module) + workspace_settings = CPAWorkspacePanel.bound_settings(module, cls) location_value = optional_setting_value(module, cls.location_object_setting) filter_count_value = optional_setting_value( module, @@ -454,7 +458,7 @@ def postprocess_bound_settings( bound = bound.with_consumed_settings( cls.property_filter_count_setting, ) - return bound.with_consumed_settings( + return bound.with_kwargs(workspace_settings).with_consumed_settings( cls.aggregate_well_mean_setting, cls.aggregate_well_median_setting, cls.aggregate_well_standard_deviation_setting, @@ -522,10 +526,10 @@ def finalize_module_blocks_for_invocation( cellprofiler_setting_literal(False), ), ModuleSetting(cls.overwrite_mode_setting.canonical, "Never"), - ModuleSetting(cls.workspace_measurement_count_setting.canonical, "0"), - ModuleSetting( - cls.wants_workspace_file_setting.canonical, - cellprofiler_setting_literal(False), + *CPAWorkspacePanel.setting_records( + cls, + arguments.arguments["wants_workspace_file"], + arguments.arguments["workspace_panels"], ), *cls._selected_object_setting_records(selected_objects), *cls._property_image_setting_records(image_channels), @@ -721,54 +725,6 @@ def _group_fields(cls, module: ModuleBlock) -> tuple[tuple[str, str], ...]: for name, statement in zip(names, columns, strict=True) ) - @classmethod - def _validate_workspace_settings( - cls, - module: ModuleBlock, - ) -> None: - count_value = optional_setting_value( - module, - cls.workspace_measurement_count_setting, - ) - declared_count = 0 if count_value is None else int(count_value) - display_tools = module.get_setting_values( - cls.workspace_display_tool_setting.canonical - ) - x_types = module.get_setting_values(cls.workspace_x_type_setting.canonical) - object_names = module.get_setting_values( - cls.workspace_object_name_setting.canonical - ) - x_measurements = module.get_setting_values( - cls.workspace_x_measurement_setting.canonical - ) - x_indices = module.get_setting_values(cls.workspace_x_index_setting.canonical) - y_types = module.get_setting_values(cls.workspace_y_type_setting.canonical) - y_measurements = module.get_setting_values( - cls.workspace_y_measurement_setting.canonical - ) - y_indices = module.get_setting_values(cls.workspace_y_index_setting.canonical) - for setting_name, values in ( - (cls.workspace_display_tool_setting, display_tools), - (cls.workspace_x_type_setting, x_types), - (cls.workspace_x_measurement_setting, x_measurements), - (cls.workspace_x_index_setting, x_indices), - (cls.workspace_y_type_setting, y_types), - (cls.workspace_y_measurement_setting, y_measurements), - (cls.workspace_y_index_setting, y_indices), - ): - cls._require_record_count(declared_count, setting_name, values) - cls._require_record_count( - declared_count * 2, - cls.workspace_object_name_setting, - object_names, - ) - wants_workspace_value = optional_setting_value( - module, - cls.wants_workspace_file_setting, - ) - if wants_workspace_value is not None: - parse_cellprofiler_bool(wants_workspace_value) - @classmethod def _require_record_count( cls, @@ -898,6 +854,8 @@ def export_to_database( object_table_mode: CellProfilerObjectTableMode = CellProfilerObjectTableMode.PER_OBJECT, selected_objects: tuple[str, ...] | None = None, wants_properties_file: bool = True, + wants_workspace_file: bool = False, + workspace_panels: tuple[CPAWorkspacePanel, ...] = (), wants_relationship_tables: bool = False, include_all_images: bool = True, image_channels: tuple[CPAImageChannelSpec, ...] = (), @@ -941,6 +899,8 @@ def export_to_database( object_table_mode=object_table_mode, selected_objects=selected_objects, wants_properties_file=wants_properties_file, + wants_workspace_file=wants_workspace_file, + workspace_panels=workspace_panels, wants_relationship_tables=wants_relationship_tables, maximum_column_name_length=maximum_column_name_length, location_object=location_object, @@ -981,14 +941,17 @@ def export_to_database( bundle: dict[str, bytes | str] = { settings.sqlite_file: CPASQLiteRenderer(dialect).render(projection, settings) } - for properties_file in CPAPropertiesRenderer(dialect).render( - settings, - resolved_channels, - projection, - ): - if properties_file.file_name in bundle: - raise ValueError( - f"ExportToDatabase emits duplicate file {properties_file.file_name!r}." + text_files = ( + *( + (file.file_name, file.text) + for file in CPAPropertiesRenderer(dialect).render( + settings, resolved_channels, projection ) - bundle[properties_file.file_name] = properties_file.text + ), + *CPAWorkspaceRenderer(dialect).render(settings).items(), + ) + for file_name, text in text_files: + if file_name in bundle: + raise ValueError(f"ExportToDatabase emits duplicate file {file_name!r}.") + bundle[file_name] = text return bundle diff --git a/tests/fixtures/cellprofiler/quality_control_4281.workspace b/tests/fixtures/cellprofiler/quality_control_4281.workspace new file mode 100644 index 0000000000..e51c9326f4 --- /dev/null +++ b/tests/fixtures/cellprofiler/quality_control_4281.workspace @@ -0,0 +1,71 @@ +CellProfiler Analyst workflow +version: 1 +CP version : 4281 + +Scatter + x-axis: ImageNumber + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PowerLogLogSlope_OrigER + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: ImageNumber + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PowerLogLogSlope_OrigHoechst + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: ImageNumber + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PowerLogLogSlope_OrigMito + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: ImageNumber + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PowerLogLogSlope_OrigPh_golgi + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: ImageNumber + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PowerLogLogSlope_OrigSyto + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: Image_ImageQuality_StdIntensity_OrigSyto + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PercentMaximal_OrigSyto + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: Image_ImageQuality_StdIntensity_OrigPh_golgi + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PercentMaximal_OrigPh_golgi + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: Image_ImageQuality_StdIntensity_OrigMito + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PercentMaximal_OrigMito + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: Image_ImageQuality_StdIntensity_OrigHoechst + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PercentMaximal_OrigHoechst + y-table: BBBC022QC_Per_Image + +Scatter + x-axis: Image_ImageQuality_StdIntensity_OrigER + x-table: BBBC022QC_Per_Image + y-axis: Image_ImageQuality_PercentMaximal_OrigER + y-table: BBBC022QC_Per_Image + +Histogram + x-axis: Image_ImageQuality_ThresholdOtsu_OrigHoechst_2W + table: BBBC022QC_Per_Image + +Histogram + x-axis: Image_ImageQuality_ThresholdOtsu_OrigPh_golgi_3FW + table: BBBC022QC_Per_Image diff --git a/tests/unit/test_cellprofiler_analyst_export.py b/tests/unit/test_cellprofiler_analyst_export.py index 7c2ee83497..b1cd83e42e 100644 --- a/tests/unit/test_cellprofiler_analyst_export.py +++ b/tests/unit/test_cellprofiler_analyst_export.py @@ -1536,7 +1536,10 @@ def test_module_contract_selects_ordered_tables_and_cpa_images_and_declares_bund ) -def test_raw_callable_uses_batch_source_plan_with_sibling_plate_step() -> None: +@pytest.mark.parametrize("workspace", [False, True]) +def test_raw_callable_uses_batch_source_plan_with_sibling_plate_step( + workspace: bool, +) -> None: store = RuntimeValueStore() measurements = _record_measurements( store, @@ -1575,6 +1578,22 @@ def test_raw_callable_uses_batch_source_plan_with_sibling_plate_step() -> None: ), ) + from openhcs.interop.cellprofiler.workspace_export import ( + CPAWorkspacePanel, + CPAWorkspaceAxis, + ) + + panels = ( + ( + CPAWorkspacePanel.from_settings( + "Histogram", + CPAWorkspaceAxis.from_settings("Image", "None", "Count", "ImageNumber"), + CPAWorkspaceAxis.from_settings("Image", "None", "None", "ImageNumber"), + ), + ) + if workspace + else () + ) bundle = export_to_database( artifact_batch=batch, context=context, @@ -1582,9 +1601,19 @@ def test_raw_callable_uses_batch_source_plan_with_sibling_plate_step() -> None: experiment_name="Example", add_table_prefix=True, table_prefix="CPA_", + wants_workspace_file=workspace, + workspace_panels=panels, ) - assert tuple(bundle) == ("analysis.sqlite", "analysis_CPA.properties") + assert tuple(bundle) == ( + ("analysis.sqlite", "analysis_CPA.properties", "analysis_CPA.workspace") + if workspace + else ("analysis.sqlite", "analysis_CPA.properties") + ) + if workspace: + assert CPAWorkspacePanel.parse_workspace(bundle["analysis_CPA.workspace"]) == ( + ("Histogram", (("x-axis", "Image_Count"), ("table", "CPA_Per_Image"))), + ) assert isinstance(bundle["analysis.sqlite"], bytes) assert str(bundle["analysis_CPA.properties"]).startswith("db_type = sqlite\n") diff --git a/tests/unit/test_cellprofiler_export_to_database.py b/tests/unit/test_cellprofiler_export_to_database.py index 18a8a28398..5823725774 100644 --- a/tests/unit/test_cellprofiler_export_to_database.py +++ b/tests/unit/test_cellprofiler_export_to_database.py @@ -142,7 +142,6 @@ "wants_filter_fields", "create_plate_filters", "overwrite_mode", - "wants_workspace_file", "workspace_measurements", } diff --git a/tests/unit/test_cellprofiler_workspace_export.py b/tests/unit/test_cellprofiler_workspace_export.py new file mode 100644 index 0000000000..0e91895cce --- /dev/null +++ b/tests/unit/test_cellprofiler_workspace_export.py @@ -0,0 +1,239 @@ +from dataclasses import replace +from pathlib import Path + +import pytest + +from benchmark.cellprofiler_export_equivalence import ( + cellprofiler_database_export_equivalence, +) +from openhcs.core.equivalence.policy import RuntimeEquivalencePolicy +from openhcs.core.runtime_exports import RuntimeExportObservation +from openhcs.interop.cellprofiler.analyst_export import ( + CellProfilerDatabaseExportSettings, + CellProfilerObjectTableMode, +) +from openhcs.interop.cellprofiler.database_column_dialect import ( + CellProfilerDatabaseColumnDialect, +) +from openhcs.interop.cellprofiler.workspace_export import ( + CPAWorkspaceAxis, + CPAWorkspacePanel, + CPAWorkspaceRenderer, +) +from openhcs.processing.backends.cellprofiler.export_to_database import ( + ExportToDatabaseModule, +) +from openhcs.interop.cellprofiler.parser import ModuleBlock + + +def settings(**kwargs): + return replace( + CellProfilerDatabaseExportSettings( + sqlite_file="Database.db", + experiment_name="Experiment", + table_prefix="Prefix_", + object_table_mode=CellProfilerObjectTableMode.PER_OBJECT, + selected_objects=None, + wants_properties_file=False, + wants_relationship_tables=False, + ), + **kwargs, + ) + + +def axis(kind="Image", name="Intensity_MeanIntensity_DNA", index="ImageNumber"): + return CPAWorkspaceAxis.from_settings(kind, "Nuclei", name, index) + + +@pytest.mark.parametrize( + "tool,display,fields", + [ + ("ScatterPlot", "Scatter", ("x-axis", "x-table", "y-axis", "y-table")), + ("DensityPlot", "Density", ("x-axis", "x-table", "y-axis", "y-table")), + ("Histogram", "Histogram", ("x-axis", "table")), + ("BoxPlot", "BoxPlot", ("x-axis", "table")), + ("PlateViewer", "PlateViewer", ("measurement", "table")), + ], +) +def test_all_native_tools_preserve_axis_roles(tool, display, fields): + panel = CPAWorkspacePanel.from_settings(tool, axis("Index"), axis("Object")) + declaration = settings(wants_workspace_file=True, workspace_panels=(panel,)) + ((name, text),) = ( + CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect("Prefix_")) + .render(declaration) + .items() + ) + assert name == "Database_Prefix.workspace" + ((tool_name, rows),) = CPAWorkspacePanel.parse_workspace(text) + assert tool_name == display + assert tuple(key for key, value in rows) == fields + assert rows[:2] == ((fields[0], "ImageNumber"), (fields[1], "Prefix_Per_Image")) + if tool in ("ScatterPlot", "DensityPlot"): + assert rows[2:] == ( + ("y-axis", "Nuclei_Intensity_MeanIntensity_DNA"), + ("y-table", "Prefix_Per_Object"), + ) + + +def test_single_axis_tools_do_not_validate_inactive_y(): + panel = CPAWorkspacePanel.from_settings( + "Histogram", axis(), axis(name="invalid\nfield") + ) + assert ( + panel.rows(CellProfilerDatabaseColumnDialect())[0][1] + == "Image_Intensity_MeanIntensity_DNA" + ) + with pytest.raises(ValueError, match="single nonempty field"): + CPAWorkspacePanel.from_settings( + "ScatterPlot", axis(), axis(name="invalid\nfield") + ) + + +def test_disabled_workspace_emits_nothing(): + assert ( + CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect()).render(settings()) + == {} + ) + + +@pytest.mark.parametrize("kind,index", [("Bogus", "ImageNumber"), ("Index", "Bogus")]) +def test_malformed_active_axes_reject(kind, index): + with pytest.raises(ValueError): + CPAWorkspacePanel.from_settings("Histogram", axis(kind, index=index), axis()) + + +def test_workspace_module_records_round_trip_without_reordering(): + panels = ( + CPAWorkspacePanel.from_settings( + "ScatterPlot", axis("Index", index="Group_Index"), axis("Object") + ), + CPAWorkspacePanel.from_settings( + "PlateViewer", axis(), axis(name="invalid\nfield") + ), + ) + records = CPAWorkspacePanel.setting_records(ExportToDatabaseModule, True, panels) + block = ModuleBlock( + name="ExportToDatabase", + module_num=1, + metadata={"variable_revision_number": 28}, + enabled=True, + setting_records=list(records), + ) + assert CPAWorkspacePanel.bound_settings(block, ExportToDatabaseModule) == { + "wants_workspace_file": True, + "workspace_panels": panels, + } + with pytest.raises(ValueError): + CPAWorkspacePanel.bound_settings( + replace(block, setting_records=list(records[:-1])), ExportToDatabaseModule + ) + + +@pytest.mark.parametrize( + "replacement", + [ + "Unknown", + "Scatter\n\tx-axis: X", + "Scatter\n\tx-axis: X\n\tx-axis: Y\n\ty-axis: Z\n\ty-table: T", + ], +) +def test_workspace_reader_rejects_unknown_missing_duplicate_fields(replacement): + with pytest.raises(ValueError): + CPAWorkspacePanel.parse_workspace( + "CellProfiler Analyst workflow\nversion: 1\nCP version : 4281\n\n" + + replacement + + "\n" + ) + + +def test_workspace_comparison_covers_order_values_and_missing_files(tmp_path): + native = ( + Path(__file__).parents[1] + / "fixtures/cellprofiler/quality_control_4281.workspace" + ).read_text() + reference = tmp_path / "reference" + reference.mkdir() + candidate = tmp_path / "candidate" + candidate.mkdir() + (reference / "QC.workspace").write_text(native) + (candidate / "QC.workspace").write_text(native) + policy = RuntimeEquivalencePolicy() + compare = lambda: cellprofiler_database_export_equivalence( + reference, + RuntimeExportObservation.from_output_roots((candidate,)), + policy=policy, + ) + assert compare().is_equivalent + (candidate / "QC.workspace").write_text( + native.replace("ImageNumber", "Group_Index", 1) + ) + assert not compare().is_equivalent + (candidate / "QC.workspace").unlink() + assert not compare().is_equivalent + + +def test_native_quality_control_fixture_contains_twelve_ordered_panels(): + native = ( + Path(__file__).parents[1] + / "fixtures/cellprofiler/quality_control_4281.workspace" + ).read_text() + panels = CPAWorkspacePanel.parse_workspace(native) + assert ( + tuple(tool for tool, rows in panels) == ("Scatter",) * 10 + ("Histogram",) * 2 + ) + assert panels[0][1][2][1] == "Image_ImageQuality_PowerLogLogSlope_OrigER" + assert panels[-1][1][0][1] == "Image_ImageQuality_ThresholdOtsu_OrigPh_golgi_3FW" + + +def test_workspace_panels_survive_source_transport(): + import openhcs.serialization.pycodify_formatters + from pycodify import Assignment, generate_python_source + + panels = ( + CPAWorkspacePanel.from_settings("ScatterPlot", axis("Index"), axis("Object")), + ) + source = generate_python_source(Assignment("panels", panels), clean_mode=True) + namespace = {} + exec(source, namespace) + assert namespace["panels"] == panels + + +def test_workspace_dialect_must_match_its_export_request(): + panel = CPAWorkspacePanel.from_settings("Histogram", axis(), axis()) + with pytest.raises(ValueError, match="table prefix"): + CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect("Other_")).render( + settings(wants_workspace_file=True, workspace_panels=(panel,)) + ) + + +def test_workspace_filename_preserves_native_relative_directory_and_single_suffix(): + panel = CPAWorkspacePanel.from_settings("Histogram", axis(), axis()) + declaration = settings( + sqlite_file="nested/Database.sqlite", + table_prefix="Prefix__", + wants_workspace_file=True, + workspace_panels=(panel,), + ) + assert tuple( + CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect("Prefix__")).render( + declaration + ) + ) == ("nested/Database_Prefix_.workspace",) + + +def test_workspace_reader_rejects_malformed_version_and_reordered_fields(): + native = ( + Path(__file__).parents[1] + / "fixtures/cellprofiler/quality_control_4281.workspace" + ).read_text() + for malformed in ( + native.replace("version: 1", "version: 2"), + native.replace("CP version : 4281", "CP version : unknown"), + native.replace( + "\tx-axis: ImageNumber\n\tx-table: BBBC022QC_Per_Image", + "\tx-table: BBBC022QC_Per_Image\n\tx-axis: ImageNumber", + 1, + ), + ): + with pytest.raises(ValueError): + CPAWorkspacePanel.parse_workspace(malformed) From 2a5b691ab1e8c0bc2dd202d089b403645112d88f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:41:10 -0400 Subject: [PATCH 091/125] refactor(equivalence): own saved correlation admission (refs #479) --- benchmark/matched_cellprofiler_batch.py | 13 ++++------- openhcs/core/equivalence/relationships.py | 22 +++++-------------- openhcs/core/equivalence/tables.py | 6 +++++ openhcs/core/runtime_equivalence.py | 10 +++++++++ openhcs/core/runtime_relationships.py | 6 +++++ .../test_exported_relationship_equivalence.py | 20 +++++++++++++++++ 6 files changed, 51 insertions(+), 26 deletions(-) diff --git a/benchmark/matched_cellprofiler_batch.py b/benchmark/matched_cellprofiler_batch.py index 77845e4437..f4891b5d07 100644 --- a/benchmark/matched_cellprofiler_batch.py +++ b/benchmark/matched_cellprofiler_batch.py @@ -190,15 +190,10 @@ def _require_compared_output_inventory( RuntimeMeasurementSnapshot.from_output_snapshot(snapshot) for snapshot in (reference_snapshot, candidate_snapshot) ) - if any( - measurement.correlated_relationships is None for measurement in measurements - ): - raise RuntimeError( - "Matched saved output inventory requires known relationship correlations." - ) - if measurements[0].relationship_differences( - measurements[1].correlated_relationships - ): + correlations = tuple( + measurement.required_relationship_correlations() for measurement in measurements + ) + if correlations[0] != correlations[1]: raise RuntimeError("Matched saved output relationship correlations differ.") comparable_tables = tuple( tuple( diff --git a/openhcs/core/equivalence/relationships.py b/openhcs/core/equivalence/relationships.py index 6b77b7717f..4dbff95e2b 100644 --- a/openhcs/core/equivalence/relationships.py +++ b/openhcs/core/equivalence/relationships.py @@ -1115,8 +1115,6 @@ def from_table( policy: RuntimeEquivalencePolicy, image_number_offset: RuntimeImageNumberOffset, ) -> tuple[Self, ...]: - if not table.rows: - raise ValueError("Relationship table has no determining declaration rows.") normalized_header = tuple( normalize_runtime_identifier(name) for name in table.header ) @@ -1125,7 +1123,7 @@ def from_table( groups: dict[ ObjectRelationshipDeclaration, list[RuntimeMeasurementRowMapping] ] = {} - for values in table.rows: + for values in table.required_rows(): row = RuntimeMeasurementRowMapping( dict(zip(table.header, values, strict=True)) ) @@ -1159,11 +1157,7 @@ def from_table( ) slice_index = image_number_offset.object_instance_key( row.row, 1 - ).slice_index - if slice_index is None: - raise ValueError( - "Relationship rows require an explicit image or slice identity." - ) + ).required_slice_index() raw_count = row.first_value(("slice_count",)) count = MeasurementScalarLiteral(raw_count).integer_value if raw_count is not None and count is None: @@ -1287,10 +1281,7 @@ def object_rows( f"Relationship endpoint {subject.name!r} lacks an object identity." ) key = image_number_offset.object_instance_key(row.row, object_id) - if key.slice_index is None: - raise ValueError( - "Relationship object rows require explicit image identities." - ) + key.required_slice_index() if key in rows and rows[key].row != row.row: raise ValueError( f"Relationship endpoint {subject.name!r} has conflicting object rows for {key!r}." @@ -1450,14 +1441,11 @@ def correlated_object_relationships( child_key = image_number_offset.object_instance_key( row.row, child_id ) - if child_key.slice_index is None: - raise ValueError( - "Relationship measurements require explicit image identities." - ) + slice_index = child_key.required_slice_index() if parent_id > 0: values.append( ( - ObjectInstanceKey(parent_id, child_key.slice_index), + ObjectInstanceKey(parent_id, slice_index), child_key, ) ) diff --git a/openhcs/core/equivalence/tables.py b/openhcs/core/equivalence/tables.py index 05962edc89..4f947d719e 100644 --- a/openhcs/core/equivalence/tables.py +++ b/openhcs/core/equivalence/tables.py @@ -78,6 +78,12 @@ class RuntimeTableSnapshot: rows: tuple[tuple[str, ...], ...] column_context: tuple[str | None, ...] = () + def required_rows(self) -> tuple[tuple[str, ...], ...]: + """Admit actual rows when a consumer needs a determining declaration.""" + if not self.rows: + raise ValueError(f"Runtime table {self.path} has no determining declaration rows.") + return self.rows + @property def participates_in_comparison(self) -> bool: """Compare output tables except the engine's Experiment key/value receipt.""" diff --git a/openhcs/core/runtime_equivalence.py b/openhcs/core/runtime_equivalence.py index f83d3f4251..4fbbbe3d2f 100644 --- a/openhcs/core/runtime_equivalence.py +++ b/openhcs/core/runtime_equivalence.py @@ -1815,6 +1815,16 @@ def __post_init__(self) -> None: def is_empty(self) -> bool: return not self.measurement_fact_counts + def required_relationship_correlations( + self, + ) -> Mapping[RuntimeMeasurementFeatureKey, ObjectInstanceRelationship]: + """Admit this snapshot for a full saved comparison, not value-only scope.""" + if self.correlated_relationships is None: + raise RuntimeError( + "Matched saved output inventory requires known relationship correlations." + ) + return self.correlated_relationships + def relationship_differences( self, candidate: ( diff --git a/openhcs/core/runtime_relationships.py b/openhcs/core/runtime_relationships.py index ea3b158552..dfadd0035c 100644 --- a/openhcs/core/runtime_relationships.py +++ b/openhcs/core/runtime_relationships.py @@ -951,6 +951,12 @@ def __post_init__(self) -> None: object.__setattr__(self, "object_id", object_id) object.__setattr__(self, "slice_index", slice_index) + def required_slice_index(self) -> int: + """Admit this object identity for a comparison requiring an explicit plane.""" + if self.slice_index is None: + raise ValueError("Object instance requires an explicit image or slice identity.") + return self.slice_index + @classmethod def from_measurement_row( cls, diff --git a/tests/unit/test_exported_relationship_equivalence.py b/tests/unit/test_exported_relationship_equivalence.py index 60273f2fac..6eeba55b94 100644 --- a/tests/unit/test_exported_relationship_equivalence.py +++ b/tests/unit/test_exported_relationship_equivalence.py @@ -391,3 +391,23 @@ def test_snapshot_json_cache_retains_correlation_scope(tmp_path: Path) -> None: ) assert restored.correlated_relationships == snapshot.correlated_relationships assert restored.measurement_fact_counts == snapshot.measurement_fact_counts + + +@pytest.mark.parametrize("slice_index", (None, 0, 2)) +def test_object_identity_owns_required_plane_admission(slice_index: int | None) -> None: + from openhcs.core.runtime_relationships import ObjectInstanceKey + + key = ObjectInstanceKey(1, slice_index) + if slice_index is None: + with pytest.raises(ValueError, match="explicit image or slice identity"): + key.required_slice_index() + else: + assert key.required_slice_index() == slice_index + + +def test_full_saved_comparison_cannot_admit_unknown_relationship_scope() -> None: + unknown = RuntimeMeasurementSnapshot({}) + with pytest.raises(RuntimeError, match="requires known relationship correlations"): + unknown.required_relationship_correlations() + known = RuntimeMeasurementSnapshot({}, {}) + assert known.required_relationship_correlations() is known.correlated_relationships From f03a7025295485508e7e7e701960baf1b9903b9d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:41:17 -0400 Subject: [PATCH 092/125] Keep workspace file selection on the existing export request (refs #480) --- docs/development/cpa_workspace_export_480.md | 2 +- .../interop/cellprofiler/analyst_export.py | 25 +++++++++++- .../interop/cellprofiler/workspace_export.py | 39 +++++++------------ .../cellprofiler/export_to_database.py | 9 +---- .../test_cellprofiler_workspace_export.py | 26 +++++-------- 5 files changed, 52 insertions(+), 49 deletions(-) diff --git a/docs/development/cpa_workspace_export_480.md b/docs/development/cpa_workspace_export_480.md index 44e67d55fd..360b8e5af7 100644 --- a/docs/development/cpa_workspace_export_480.md +++ b/docs/development/cpa_workspace_export_480.md @@ -4,7 +4,7 @@ ExportToDatabase accepted and validated workspace panel settings, then discarded The existing export settings now retain the requested workspace and original ordered panel declarations. `CPAWorkspacePanel` owns the external panel grammar, repeated setting binding and authoring transport. Its Scatter/Density leaves share two-axis rendering; Histogram/Box/PlateViewer use the single-axis grammar, with PlateViewer's native `measurement` field. `CPAWorkspaceAxis` derives its Image/Object/Index implementations from their declarations. Image/object measurements use the existing `MeasurementSubject` and database dialect; there is no parallel tool enum or handwritten dispatch roster. -The public exporter merges workspace text into the existing file bundle and retains its duplicate-file rejection. The shared database-export comparison parses ordered workspace semantics, rejects invalid layout or unknown fields/tools, and reports missing or changed panels. The strict matched inventory now admits `.workspace` because it has an actual comparison consumer. Database and properties rendering remain unchanged. +The existing export settings own enabled-workspace selection, exact filenames and matching table-prefix admission. The format renderer consumes only the resulting ordered panels. Authoring projection binds the already-validated public call keywords at the panel owner, retaining unrelated export settings without copying two raw key lookups. The public exporter merges workspace text into the existing file bundle and retains its duplicate-file rejection. The shared database-export comparison parses ordered workspace semantics, rejects invalid layout or unknown fields/tools, and reports missing or changed panels. The strict matched inventory now admits `.workspace` because it has an actual comparison consumer. Database and properties rendering remain unchanged. Native 4.2.8.1 emits raw subject/feature names in workspace axes, and Object axes always name `Per_Object`, even when separate object tables are selected. These native quirks are preserved, not silently translated to different scientific columns. The near-term QC fixture has only Image/Index axes: both retained native workspace texts match exactly, and all 22 axis references resolve to actual native SQLite columns. This does not establish arbitrary native object-workspace references or shortened column names are valid. The reader validates workflow format version 1 and the numeric producer-version header; the CP producer version is descriptive rather than compared panel meaning. diff --git a/openhcs/interop/cellprofiler/analyst_export.py b/openhcs/interop/cellprofiler/analyst_export.py index 9ceba032f7..db6c084f78 100644 --- a/openhcs/interop/cellprofiler/analyst_export.py +++ b/openhcs/interop/cellprofiler/analyst_export.py @@ -6,7 +6,10 @@ from __future__ import annotations -from openhcs.interop.cellprofiler.workspace_export import CPAWorkspacePanel +from openhcs.interop.cellprofiler.workspace_export import ( + CPAWorkspacePanel, + CPAWorkspaceRenderer, +) from base64 import b64encode from collections import defaultdict @@ -396,6 +399,26 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "thumbnail_image_names", thumbnail_names) + def workspace_files( + self, dialect: CellProfilerDatabaseColumnDialect + ) -> dict[str, str]: + """Resolve enabled workspace filenames from this exact export request.""" + if not self.wants_workspace_file: + return {} + if dialect.table_prefix != self.table_prefix: + raise ValueError( + "CPA workspace dialect disagrees with export table prefix." + ) + name = str(Path(self.sqlite_file).with_suffix("")) + prefix = self.table_prefix.removesuffix("_") + if prefix: + name = f"{name}_{prefix}" + return { + f"{name}.workspace": CPAWorkspaceRenderer(dialect).render( + self.workspace_panels + ) + } + def exports_object(self, object_name: str) -> bool: """Return whether this export includes one declared object subject.""" diff --git a/openhcs/interop/cellprofiler/workspace_export.py b/openhcs/interop/cellprofiler/workspace_export.py index c32a9a4670..3de9d0804a 100644 --- a/openhcs/interop/cellprofiler/workspace_export.py +++ b/openhcs/interop/cellprofiler/workspace_export.py @@ -4,7 +4,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from pathlib import Path from typing import ClassVar, TYPE_CHECKING from metaclass_registry import AutoRegisterMeta @@ -24,9 +23,6 @@ from openhcs.interop.cellprofiler.settings_binder import parse_cellprofiler_bool if TYPE_CHECKING: - from openhcs.interop.cellprofiler.analyst_export import ( - CellProfilerDatabaseExportSettings, - ) from openhcs.processing.backends.cellprofiler.export_to_database import ( ExportToDatabaseModule, ) @@ -182,20 +178,24 @@ def bound_settings( def setting_records( cls, declaration: type[ExportToDatabaseModule], - wants: bool, - panels: tuple[CPAWorkspacePanel, ...], + *, + wants_workspace_file: bool, + workspace_panels: tuple[CPAWorkspacePanel, ...], + **other_export_parameters: object, ) -> tuple[ModuleSetting, ...]: + """Project workspace records from the already-bound public export call.""" + del other_export_parameters records = [ ModuleSetting( declaration.workspace_measurement_count_setting.canonical, - str(len(panels)), + str(len(workspace_panels)), ), ModuleSetting( declaration.wants_workspace_file_setting.canonical, - cellprofiler_setting_literal(wants), + cellprofiler_setting_literal(wants_workspace_file), ), ] - for panel in panels: + for panel in workspace_panels: values = ( (declaration.workspace_display_tool_setting, panel.setting_name), (declaration.workspace_x_type_setting, panel.x.measurement_type), @@ -334,26 +334,17 @@ class CPAWorkspaceRenderer: dialect: CellProfilerDatabaseColumnDialect - def render(self, settings: CellProfilerDatabaseExportSettings) -> dict[str, str]: - if not settings.wants_workspace_file: - return {} - if self.dialect.table_prefix != settings.table_prefix: - raise ValueError( - "CPA workspace dialect disagrees with export table prefix." - ) - name = str(Path(settings.sqlite_file).with_suffix("")) - prefix = settings.table_prefix.removesuffix("_") - if prefix: - name = f"{name}_{prefix}" + def render(self, panels: tuple[CPAWorkspacePanel, ...]) -> str: + """Encode panel meaning without deciding whether an export requested it.""" sections = tuple( "\n" + panel.display_name + "\n" + "\n".join(f"\t{key}: {value}" for key, value in panel.rows(self.dialect)) + "\n" - for panel in settings.workspace_panels + for panel in panels ) - return { - f"{name}.workspace": "CellProfiler Analyst workflow\nversion: 1\nCP version : 4281\n" + return ( + "CellProfiler Analyst workflow\nversion: 1\nCP version : 4281\n" + "".join(sections) - } + ) diff --git a/openhcs/processing/backends/cellprofiler/export_to_database.py b/openhcs/processing/backends/cellprofiler/export_to_database.py index 9a650ab9c9..9b144f20d4 100644 --- a/openhcs/processing/backends/cellprofiler/export_to_database.py +++ b/openhcs/processing/backends/cellprofiler/export_to_database.py @@ -34,7 +34,6 @@ ) from openhcs.interop.cellprofiler.workspace_export import ( CPAWorkspacePanel, - CPAWorkspaceRenderer, ) from openhcs.interop.cellprofiler.cellprofiler_literals import ( cellprofiler_setting_literal, @@ -526,11 +525,7 @@ def finalize_module_blocks_for_invocation( cellprofiler_setting_literal(False), ), ModuleSetting(cls.overwrite_mode_setting.canonical, "Never"), - *CPAWorkspacePanel.setting_records( - cls, - arguments.arguments["wants_workspace_file"], - arguments.arguments["workspace_panels"], - ), + *CPAWorkspacePanel.setting_records(cls, **arguments.arguments), *cls._selected_object_setting_records(selected_objects), *cls._property_image_setting_records(image_channels), *cls._group_setting_records(group_fields), @@ -948,7 +943,7 @@ def export_to_database( settings, resolved_channels, projection ) ), - *CPAWorkspaceRenderer(dialect).render(settings).items(), + *settings.workspace_files(dialect).items(), ) for file_name, text in text_files: if file_name in bundle: diff --git a/tests/unit/test_cellprofiler_workspace_export.py b/tests/unit/test_cellprofiler_workspace_export.py index 0e91895cce..4f5616a23f 100644 --- a/tests/unit/test_cellprofiler_workspace_export.py +++ b/tests/unit/test_cellprofiler_workspace_export.py @@ -18,7 +18,6 @@ from openhcs.interop.cellprofiler.workspace_export import ( CPAWorkspaceAxis, CPAWorkspacePanel, - CPAWorkspaceRenderer, ) from openhcs.processing.backends.cellprofiler.export_to_database import ( ExportToDatabaseModule, @@ -58,11 +57,9 @@ def axis(kind="Image", name="Intensity_MeanIntensity_DNA", index="ImageNumber"): def test_all_native_tools_preserve_axis_roles(tool, display, fields): panel = CPAWorkspacePanel.from_settings(tool, axis("Index"), axis("Object")) declaration = settings(wants_workspace_file=True, workspace_panels=(panel,)) - ((name, text),) = ( - CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect("Prefix_")) - .render(declaration) - .items() - ) + ((name, text),) = declaration.workspace_files( + CellProfilerDatabaseColumnDialect("Prefix_") + ).items() assert name == "Database_Prefix.workspace" ((tool_name, rows),) = CPAWorkspacePanel.parse_workspace(text) assert tool_name == display @@ -90,10 +87,7 @@ def test_single_axis_tools_do_not_validate_inactive_y(): def test_disabled_workspace_emits_nothing(): - assert ( - CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect()).render(settings()) - == {} - ) + assert settings().workspace_files(CellProfilerDatabaseColumnDialect()) == {} @pytest.mark.parametrize("kind,index", [("Bogus", "ImageNumber"), ("Index", "Bogus")]) @@ -111,7 +105,9 @@ def test_workspace_module_records_round_trip_without_reordering(): "PlateViewer", axis(), axis(name="invalid\nfield") ), ) - records = CPAWorkspacePanel.setting_records(ExportToDatabaseModule, True, panels) + records = CPAWorkspacePanel.setting_records( + ExportToDatabaseModule, wants_workspace_file=True, workspace_panels=panels + ) block = ModuleBlock( name="ExportToDatabase", module_num=1, @@ -201,8 +197,8 @@ def test_workspace_panels_survive_source_transport(): def test_workspace_dialect_must_match_its_export_request(): panel = CPAWorkspacePanel.from_settings("Histogram", axis(), axis()) with pytest.raises(ValueError, match="table prefix"): - CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect("Other_")).render( - settings(wants_workspace_file=True, workspace_panels=(panel,)) + settings(wants_workspace_file=True, workspace_panels=(panel,)).workspace_files( + CellProfilerDatabaseColumnDialect("Other_") ) @@ -215,9 +211,7 @@ def test_workspace_filename_preserves_native_relative_directory_and_single_suffi workspace_panels=(panel,), ) assert tuple( - CPAWorkspaceRenderer(CellProfilerDatabaseColumnDialect("Prefix__")).render( - declaration - ) + declaration.workspace_files(CellProfilerDatabaseColumnDialect("Prefix__")) ) == ("nested/Database_Prefix_.workspace",) From 23cc8cf262bfd043acafd3154b390e5ba0c00c44 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:55:40 -0400 Subject: [PATCH 093/125] Derive strict CPA inventory coverage from actual comparison readers (refs #480) --- benchmark/adapters/openhcs.py | 4 +- benchmark/cellprofiler_export_equivalence.py | 69 +++++----- benchmark/matched_cellprofiler_batch.py | 9 +- docs/development/cpa_workspace_export_480.md | 4 +- openhcs/core/equivalence/report.py | 9 ++ .../test_cellprofiler_export_equivalence.py | 3 + .../test_cellprofiler_workspace_export.py | 123 ++++++++++++++++++ 7 files changed, 183 insertions(+), 38 deletions(-) diff --git a/benchmark/adapters/openhcs.py b/benchmark/adapters/openhcs.py index c5cd8be0ef..3653bc3f26 100644 --- a/benchmark/adapters/openhcs.py +++ b/benchmark/adapters/openhcs.py @@ -524,7 +524,9 @@ def _run_converted_cppipe_pipeline( ( *equivalence_report.differences, *database_export_report.differences, - ) + ), + equivalence_report.compared_output_files + | database_export_report.compared_output_files, ) if not equivalence_report.is_equivalent: equivalence_failure_message = ( diff --git a/benchmark/cellprofiler_export_equivalence.py b/benchmark/cellprofiler_export_equivalence.py index a07723be6e..139423df14 100644 --- a/benchmark/cellprofiler_export_equivalence.py +++ b/benchmark/cellprofiler_export_equivalence.py @@ -56,30 +56,29 @@ def cellprofiler_database_export_equivalence( reference_root = Path(reference_output_root) reference_properties = tuple(sorted(reference_root.rglob("*.properties"))) candidate_properties = _outputs_with_suffix(candidate_exports, ".properties") - differences = [ - *_sqlite_export_differences( - _declared_sqlite_paths( - tuple(sorted(reference_root.rglob("*.db"))), - reference_properties, - ), - _declared_sqlite_paths( - _outputs_with_suffix(candidate_exports, ".db"), - candidate_properties, - ), + reference_databases = _declared_sqlite_paths( + tuple(sorted(reference_root.rglob("*.db"))), reference_properties + ) + candidate_databases = _declared_sqlite_paths( + _outputs_with_suffix(candidate_exports, ".db"), candidate_properties + ) + reference_workspaces = tuple(sorted(reference_root.rglob("*.workspace"))) + candidate_workspaces = _outputs_with_suffix(candidate_exports, ".workspace") + reports = ( + _sqlite_export_equivalence( + reference_databases, + candidate_databases, _declared_sqlite_table_subjects(reference_properties), _declared_sqlite_table_subjects(candidate_properties), policy, ), - *_workspace_export_differences( - tuple(sorted(reference_root.rglob("*.workspace"))), - _outputs_with_suffix(candidate_exports, ".workspace"), - ), - *_properties_export_differences( - reference_properties, - candidate_properties, - ), - ] - return RuntimeEquivalenceReport(tuple(differences)) + _workspace_export_equivalence(reference_workspaces, candidate_workspaces), + _properties_export_equivalence(reference_properties, candidate_properties), + ) + return RuntimeEquivalenceReport( + tuple(difference for report in reports for difference in report.differences), + frozenset(path for report in reports for path in report.compared_output_files), + ) def cellprofiler_native_shard_equivalence( @@ -159,13 +158,15 @@ def cellprofiler_native_shard_equivalence( for root in shard_roots: shard_properties = tuple(sorted(root.rglob("*.properties"))) differences.extend( - _properties_export_differences(reference_properties, shard_properties) + _properties_export_equivalence( + reference_properties, shard_properties + ).differences ) differences.extend( - _workspace_export_differences( + _workspace_export_equivalence( tuple(sorted(reference_root.rglob("*.workspace"))), tuple(sorted(root.rglob("*.workspace"))), - ) + ).differences ) shard_subjects = _declared_sqlite_table_subjects(shard_properties) if shard_subjects != reference_subjects: @@ -282,18 +283,19 @@ def _declared_sqlite_paths( return tuple(path for path in database_paths if path.name in declared_names) -def _sqlite_export_differences( +def _sqlite_export_equivalence( reference_paths: Sequence[Path], candidate_paths: Sequence[Path], reference_subjects: Mapping[str, Mapping[str, MeasurementSubject]], candidate_subjects: Mapping[str, Mapping[str, MeasurementSubject]], policy: RuntimeEquivalencePolicy, -) -> tuple[RuntimeEquivalenceDifference, ...]: +) -> RuntimeEquivalenceReport: differences, reference_by_name, candidate_by_name = _named_output_differences( reference_paths, candidate_paths, output_label="SQLite database", ) + compared_paths: set[Path] = set() for name in sorted(reference_by_name.keys() & candidate_by_name.keys()): differences.extend( _sqlite_database_differences( @@ -304,7 +306,8 @@ def _sqlite_export_differences( policy, ) ) - return tuple(differences) + compared_paths.update((reference_by_name[name], candidate_by_name[name])) + return RuntimeEquivalenceReport(tuple(differences), frozenset(compared_paths)) def _sqlite_database_differences( @@ -597,15 +600,16 @@ def _normalized_sqlite_row( return tuple(values) -def _properties_export_differences( +def _properties_export_equivalence( reference_paths: Sequence[Path], candidate_paths: Sequence[Path], -) -> tuple[RuntimeEquivalenceDifference, ...]: +) -> RuntimeEquivalenceReport: differences, reference_by_name, candidate_by_name = _named_output_differences( reference_paths, candidate_paths, output_label="CPA properties file", ) + compared_paths: set[Path] = set() for name in sorted(reference_by_name.keys() & candidate_by_name.keys()): reference_properties = _read_cpa_properties(reference_by_name[name]) candidate_properties = _read_cpa_properties(candidate_by_name[name]) @@ -634,7 +638,8 @@ def _properties_export_differences( f"{mismatched_values!r}", ) ) - return tuple(differences) + compared_paths.update((reference_by_name[name], candidate_by_name[name])) + return RuntimeEquivalenceReport(tuple(differences), frozenset(compared_paths)) def _read_cpa_properties(path: Path) -> dict[str, str]: @@ -698,12 +703,13 @@ def _unique_paths_by_name( return by_name -def _workspace_export_differences(reference_paths, candidate_paths): +def _workspace_export_equivalence(reference_paths, candidate_paths): from openhcs.interop.cellprofiler.workspace_export import CPAWorkspacePanel differences, reference_by_name, candidate_by_name = _named_output_differences( reference_paths, candidate_paths, output_label="CPA workspace" ) + compared_paths: set[Path] = set() for name in reference_by_name.keys() & candidate_by_name.keys(): reference = CPAWorkspacePanel.parse_workspace( reference_by_name[name].read_text() @@ -718,4 +724,5 @@ def _workspace_export_differences(reference_paths, candidate_paths): f"CPA workspace {name!r} panel declarations differ.", ) ) - return tuple(differences) + compared_paths.update((reference_by_name[name], candidate_by_name[name])) + return RuntimeEquivalenceReport(tuple(differences), frozenset(compared_paths)) diff --git a/benchmark/matched_cellprofiler_batch.py b/benchmark/matched_cellprofiler_batch.py index ff7652e785..017edca9bb 100644 --- a/benchmark/matched_cellprofiler_batch.py +++ b/benchmark/matched_cellprofiler_batch.py @@ -62,6 +62,7 @@ ) from openhcs.core.equivalence.comparison import runtime_image_differences from openhcs.core.equivalence.outputs import RuntimeOutputSnapshot +from openhcs.core.equivalence.report import RuntimeEquivalenceReport from openhcs.core.source_matching import SourceImageSetIdentityPolicy from openhcs.core.equivalence.policy import normalize_runtime_identifier from openhcs.core.input_workspace import InputWorkspacePreparationRequest @@ -177,6 +178,7 @@ def _require_compared_output_inventory( reference_snapshot: RuntimeOutputSnapshot, candidate_snapshot: RuntimeOutputSnapshot, candidate_managed_files: frozenset[Path] = frozenset(), + compared_file_report: RuntimeEquivalenceReport = RuntimeEquivalenceReport(()), ) -> None: """Reject unqualified output formats and missing scientific output files.""" counts = [] @@ -193,11 +195,7 @@ def _require_compared_output_inventory( compared_files = ( frozenset(exports.table_outputs) | frozenset(exports.image_outputs) - | frozenset( - path - for path in exports.output_files - if path.suffix.lower() in {".db", ".properties", ".workspace"} - ) + | (files & compared_file_report.compared_output_files) ) if files != compared_files: raise RuntimeError( @@ -836,6 +834,7 @@ def capture_progress_event(event: Mapping[str, Any]) -> None: reference_snapshot=native_snapshot, candidate_snapshot=candidate_snapshot, candidate_managed_files=managed_output_files, + compared_file_report=database_report, ) result = { "repetition": repetition, diff --git a/docs/development/cpa_workspace_export_480.md b/docs/development/cpa_workspace_export_480.md index 360b8e5af7..924a2493d0 100644 --- a/docs/development/cpa_workspace_export_480.md +++ b/docs/development/cpa_workspace_export_480.md @@ -4,10 +4,12 @@ ExportToDatabase accepted and validated workspace panel settings, then discarded The existing export settings now retain the requested workspace and original ordered panel declarations. `CPAWorkspacePanel` owns the external panel grammar, repeated setting binding and authoring transport. Its Scatter/Density leaves share two-axis rendering; Histogram/Box/PlateViewer use the single-axis grammar, with PlateViewer's native `measurement` field. `CPAWorkspaceAxis` derives its Image/Object/Index implementations from their declarations. Image/object measurements use the existing `MeasurementSubject` and database dialect; there is no parallel tool enum or handwritten dispatch roster. -The existing export settings own enabled-workspace selection, exact filenames and matching table-prefix admission. The format renderer consumes only the resulting ordered panels. Authoring projection binds the already-validated public call keywords at the panel owner, retaining unrelated export settings without copying two raw key lookups. The public exporter merges workspace text into the existing file bundle and retains its duplicate-file rejection. The shared database-export comparison parses ordered workspace semantics, rejects invalid layout or unknown fields/tools, and reports missing or changed panels. The strict matched inventory now admits `.workspace` because it has an actual comparison consumer. Database and properties rendering remain unchanged. +The existing export settings own enabled-workspace selection, exact filenames and matching table-prefix admission. The format renderer consumes only the resulting ordered panels. Authoring projection binds the already-validated public call keywords at the panel owner, retaining unrelated export settings without copying two raw key lookups. The public exporter merges workspace text into the existing file bundle and retains its duplicate-file rejection. The shared database-export comparison parses ordered workspace semantics, rejects invalid layout or unknown fields/tools, and reports missing or changed panels. The existing equivalence report records physical inputs only after each matched SQLite, properties or workspace reader completes. The strict matched inventory consumes those exact paths instead of admitting a filename suffix roster. Unknown, unmatched and unread files remain uncovered; file consumption does not erase scientific differences. Existing CSV/image callers retain their separate declared output coverage. Database and properties rendering remain unchanged. Native 4.2.8.1 emits raw subject/feature names in workspace axes, and Object axes always name `Per_Object`, even when separate object tables are selected. These native quirks are preserved, not silently translated to different scientific columns. The near-term QC fixture has only Image/Index axes: both retained native workspace texts match exactly, and all 22 axis references resolve to actual native SQLite columns. This does not establish arbitrary native object-workspace references or shortened column names are valid. The reader validates workflow format version 1 and the numeric producer-version header; the CP producer version is descriptive rather than compared panel meaning. The retained native fixture comes from source `6e69b85122d3c1a44857835ba9b3fe77e743240d`, exact matched ten physical images/two image sets and native CP 4.2.8.1. Its SHA256 is `884f14365ed643c8dd7050ea77e416a1b7253dca83ff11d2e01aa54a8e07bdc6`. Source-backed installed native renderer: `cellprofiler/modules/exporttodatabase.py:4799`. The original native inventory red and interrupted test shard remain in `/var/tmp`; the current production pipeline still needs a fresh complete-output gate before #480 closes. Ownership evidence reuses the complete NRA class census from the shared derivation audit and the finite original CPA class inventory at `/var/tmp/openhcs-qc-workspace-owner-inventory-20261002.json`. The bounded migration removes the old validate-and-discard workspace algorithm from `ExportToDatabaseModule`; authored setting names remain authoritative there, and the panel owner consumes those exact declarations. These are feature and boundary-format changes authored as exact patches; no automatic codemod or formal scientific equivalence is claimed. + +The report consumer inventory parses 1,620 original Python modules across production, benchmark, scripts and tests; its finite AST receipt is `/var/tmp/openhcs-cpa-workspace-480-report-consumer-ast-20261002.json`. Runtime aliases and third-party serialization are not claimed closed. Existing adapter aggregation preserves consumed-path unions. Missing files, malformed readers, changed scientific panels, unknown copies and ambiguous physical aliases have separate controls; the original guard failures and malformed command invocation logs remain retained. diff --git a/openhcs/core/equivalence/report.py b/openhcs/core/equivalence/report.py index fadbc4e2d6..1122e8865f 100644 --- a/openhcs/core/equivalence/report.py +++ b/openhcs/core/equivalence/report.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from enum import Enum +from pathlib import Path class RuntimeEquivalenceDifferenceKind(str, Enum): @@ -43,6 +44,14 @@ class RuntimeEquivalenceReport: """Semantic equivalence result for two runtime outputs.""" differences: tuple[RuntimeEquivalenceDifference, ...] + compared_output_files: frozenset[Path] = frozenset() + + def __post_init__(self) -> None: + object.__setattr__( + self, + "compared_output_files", + frozenset(Path(path) for path in self.compared_output_files), + ) @property def is_equivalent(self) -> bool: diff --git a/tests/unit/test_cellprofiler_export_equivalence.py b/tests/unit/test_cellprofiler_export_equivalence.py index 6d29760c66..6417a741b8 100644 --- a/tests/unit/test_cellprofiler_export_equivalence.py +++ b/tests/unit/test_cellprofiler_export_equivalence.py @@ -352,6 +352,9 @@ def test_database_export_equivalence_compares_sqlite_and_semantic_properties( ) assert report.is_equivalent + assert report.compared_output_files == frozenset( + (*reference.iterdir(), *candidate.iterdir()) + ) def test_database_export_equivalence_rejects_unequal_row_count_before_value_projection( diff --git a/tests/unit/test_cellprofiler_workspace_export.py b/tests/unit/test_cellprofiler_workspace_export.py index 4f5616a23f..f6a0a3e13a 100644 --- a/tests/unit/test_cellprofiler_workspace_export.py +++ b/tests/unit/test_cellprofiler_workspace_export.py @@ -231,3 +231,126 @@ def test_workspace_reader_rejects_malformed_version_and_reordered_fields(): ): with pytest.raises(ValueError): CPAWorkspacePanel.parse_workspace(malformed) + + +def test_workspace_comparison_coverage_is_physical_and_separate_from_science(tmp_path): + from benchmark.matched_cellprofiler_batch import _require_compared_output_inventory + from openhcs.core.equivalence.outputs import RuntimeOutputSnapshot + + reference, candidate = tmp_path / "reference", tmp_path / "candidate" + reference.mkdir() + candidate.mkdir() + fixture = ( + Path(__file__).parents[1] + / "fixtures/cellprofiler/quality_control_4281.workspace" + ) + paths = tuple(root / "QC.workspace" for root in (reference, candidate)) + for path in paths: + path.write_bytes(fixture.read_bytes()) + exports = tuple( + RuntimeExportObservation.from_output_root(root) + for root in (reference, candidate) + ) + report = cellprofiler_database_export_equivalence( + reference, exports[1], policy=RuntimeEquivalencePolicy() + ) + assert report.is_equivalent + assert report.compared_output_files == frozenset(paths) + kwargs = dict( + reference_files=frozenset(reference.iterdir()), + candidate_files=frozenset(candidate.iterdir()), + reference_exports=exports[0], + candidate_exports=exports[1], + reference_snapshot=RuntimeOutputSnapshot(), + candidate_snapshot=RuntimeOutputSnapshot(), + ) + with pytest.raises(RuntimeError, match="without a value comparison"): + _require_compared_output_inventory(**kwargs) + _require_compared_output_inventory(**kwargs, compared_file_report=report) + paths[1].write_text(paths[1].read_text().replace("Image_", "Changed_", 1)) + changed = cellprofiler_database_export_equivalence( + reference, exports[1], policy=RuntimeEquivalencePolicy() + ) + assert not changed.is_equivalent + assert changed.compared_output_files == frozenset(paths) + _require_compared_output_inventory(**kwargs, compared_file_report=changed) + unknown = candidate / "copied-export.unknown" + unknown.write_bytes(paths[1].read_bytes()) + with pytest.raises(RuntimeError, match="without a value comparison"): + _require_compared_output_inventory( + **{**kwargs, "candidate_files": frozenset(candidate.iterdir())}, + compared_file_report=changed, + ) + + +def test_unmatched_workspace_and_reader_failures_never_claim_coverage( + tmp_path, monkeypatch +): + reference, candidate = tmp_path / "reference", tmp_path / "candidate" + reference.mkdir() + candidate.mkdir() + fixture = ( + Path(__file__).parents[1] + / "fixtures/cellprofiler/quality_control_4281.workspace" + ) + (reference / "QC.workspace").write_bytes(fixture.read_bytes()) + extra = candidate / "Unmatched.workspace" + extra.write_bytes(fixture.read_bytes()) + exports = RuntimeExportObservation.from_output_root(candidate) + report = cellprofiler_database_export_equivalence( + reference, exports, policy=RuntimeEquivalencePolicy() + ) + assert not report.is_equivalent + assert not report.compared_output_files + extra.rename(candidate / "QC.workspace") + extra = candidate / "QC.workspace" + exports = RuntimeExportObservation.from_output_root(candidate) + extra.write_text("not a CPA workspace") + with pytest.raises(ValueError): + cellprofiler_database_export_equivalence( + reference, exports, policy=RuntimeEquivalencePolicy() + ) + extra.write_bytes(fixture.read_bytes()) + original_read = Path.read_text + failure = OSError("actual reader failed") + + def fail_read(path, *args, **kwargs): + if path == extra: + raise failure + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", fail_read) + with pytest.raises(OSError) as raised: + cellprofiler_database_export_equivalence( + reference, exports, policy=RuntimeEquivalencePolicy() + ) + assert raised.value is failure + + +def test_workspace_aliases_are_compared_by_each_actual_path(tmp_path): + reference, candidate = tmp_path / "reference", tmp_path / "candidate" + reference.mkdir() + candidate.mkdir() + fixture = ( + Path(__file__).parents[1] + / "fixtures/cellprofiler/quality_control_4281.workspace" + ) + first = reference / "QC.workspace" + first.write_bytes(fixture.read_bytes()) + second = candidate / "QC.workspace" + second.symlink_to(first) + report = cellprofiler_database_export_equivalence( + reference, + RuntimeExportObservation.from_output_root(candidate), + policy=RuntimeEquivalencePolicy(), + ) + assert report.is_equivalent + assert report.compared_output_files == frozenset((first, second)) + (candidate / "other").mkdir() + (candidate / "other/QC.workspace").symlink_to(first) + with pytest.raises(ValueError, match="ambiguous"): + cellprofiler_database_export_equivalence( + reference, + RuntimeExportObservation.from_output_root(candidate), + policy=RuntimeEquivalencePolicy(), + ) From 4904884e330126916ffcc7e18b9884ab35fdce63 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 17:58:58 -0400 Subject: [PATCH 094/125] Record merged readiness fixes and strict saved-reader evidence --- .../shared_runtime_plumbing_384_20261001.rst | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index b32bf532b1..fa42e9d337 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1779,3 +1779,63 @@ changes beyond the receiving review's original four-file eb2/wheel delta. Installed receiving obligations419/433/435/450, whole-branch R0, full native image parity, full-catalog/scaling and figures remain open. A first-wave fresh four-case ordinary/native qualification is in progress, not a completed gate. + +Library readiness and saved-reader checkpoint, 2026-10-02 +------------------------------------------------------- + +Main ``b4405ff3541add6b91004002b7d6f389f1208636`` is normally merged. +Independent PR482 closes issue477 and PR486 closes issue481. Main's singleton +invocation fix passes 401 controls and original scoped R0/R1; its actual public +translocation observation is 1.679128s execution / 3.094840s total / 1.008657s +compilation. Both saved warm native repetitions pass full database, schema, +discrete and numerical measurements, properties, exact image and physical +inventory gates. This single main-based observation is not a measured gain. +Receipt: /var/tmp/openhcs-singleton-label-main-477-saved-native-science-20261002.json +(SHA256 5c5c4c497df1ee4b115aba42650941f451fac3ae3f784cfcfe3144e0ae94d6ac). + +Issue481 prepares canonical masked float32/float64 declumping signatures through +the existing morphology backend library lifecycle. Numerical bodies are +unchanged. Eight isolated controls and original scoped R0/R1 pass; cold and +disk-cache witnesses reject subsequent compilation or cache loading. Frozen +integrated ``adef79b4ff3825b35a41ef96b60f3bc421a5ddd5`` Speckles execution is +1.915174s, total 3.212164s, compilation 0.959121s. No kernel-cache writes occur +inside execution. All three scientific CSVs match the original fully native +qualified output exactly, with complete inventory and freezes. This necessary +readiness repair does not establish a speedup. Receipt: +/var/tmp/openhcs-pr394-morphology-ready-speckles-science-20261002.json +(SHA256 29526b6c5c37bbd840c706393b9290695894bc80a2247aedc7c718cb58ee5477). + +Issue479's saved relationship reader is integrated from ``2de4e021f``. It +validates directed edge declarations, object/image domains, reverse pairs and +redundant Parent/Children fields before using scalar projections. Saved +snapshots and cache transport retain endpoint correlations; typed value-only +snapshots explicitly retain UNKNOWN evidence. Both-root scoped R0 passes, +but original R1 remains RED for an unchanged validator's relocation (+1 core, +-1 interop, zero global type-check change). This is not all-guards admission. +The merged-parent controls pass 380; root integration passes 235. + +Corrected saved Beginner replay uses the actual complete output root. Both +native repetitions have exact two-image agreement, complete physical-file +coverage and five matching correlation keys / 2,093 directed pairs. Full +measurement parity remains RED: 80 extra Costes features and 63 missing Nuclei +child-mean features. The earlier helper's incorrect results-directory root and +missing-image report are rejected, retained harness evidence. Reader receipt: +/var/tmp/openhcs-relationship-479-full-saved-replay-v2-20261002/receipt.json. + +Issue487 isolates the first loss for unwanted Costes work: existing settings +postprocessing treats legacy Run all metrics=Accurate as true and overwrites +the five authored flags. Native CellProfiler uses this aggregate setting only +for UI visibility, including when Yes; individual choices remain authoritative. +The correct existing owner is ModuleOnlySettingBinding, with the obsolete +runtime override removed. Public-importer RED and native/global AST census are +retained. Qualification is in progress. Colocalization's ordinary 2.514475s +inclusive step cost inside 6.424050s execution is an upper bound, not measured +Costes-exclusive work or a promised saving. The separate 63 missing means +still require first-loss evidence. No numerical tolerance or parity gate changes. + +Compiler profiling confirms repeated effective-configuration construction and +cold schema analysis, but its timings overlap and are diagnostic only. A +private prototype is not admitted: default-factory/callback and mutable-state +laws remain unproved. This route is deferred while the larger unauthorized +metric execution is qualified. Full scaling and fresh figures remain pending; +the long-running optimization goal remains active. From 907087bb9b4c1da1656b41baea7aed65bf0c7866 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 18:15:12 -0400 Subject: [PATCH 095/125] Record complete requested CPA workspace production qualification --- .../shared_runtime_plumbing_384_20261001.rst | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index fa42e9d337..29d47c301a 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1839,3 +1839,43 @@ private prototype is not admitted: default-factory/callback and mutable-state laws remain unproved. This route is deferred while the larger unauthorized metric execution is qualified. Full scaling and fresh figures remain pending; the long-running optimization goal remains active. + +Requested CPA workspace production qualification +------------------------------------------------ + +Issue480 is integrated at ``38d02e20a795ad6f8c6457459c380995a798d0fe``. +The existing export settings retain authored workspace requests and panels; +nominal tool/axis leaves own the irreducible external grammar and derive +columns from the existing CPA dialect. The discarded settings validator is +removed. Publication uses the same existing export bundle/materialization +path. Full physical-file admission now consumes paths actually read by the +existing CPA equivalence report, instead of a manually synchronized suffix +roster. Unmatched files and failed readers cannot claim coverage. + +Original scoped R0 across all three roots and original R1 pass against2560; +138 agent controls and 81 root integration controls pass. An actual ordinary +QualityControl public run on clean, sealed38d observes 0.652361870s execution, +2.120112958s total and 1.088341713s compilation. Startup/readiness and shutdown +are excluded. No diagnostic hooks are installed. Both immutable saved native +repetitions pass database/schema/discrete/numerical measurements, nonempty +declared two-row Image table, requested workspace meaning, every physical +output and all source/input/dependency/native/controller freezes. This is a +correctness qualification, not a paired performance gain. The original V4 +missing-workspace inventory RED is retained unchanged. + +Complete receipt: +/var/tmp/openhcs-cpa-workspace-480-integrated-saved-native-v1-20261002.json +SHA256 72e22c3859081de0494ad24440d36a54b306adbaa509d8afcb9db87d671ece5d. +Ordinary source-freeze SHA256: +317de6735208ed9c99149d81296bf40dd3c1e70823e198da69b7cb7575d47e5d. +Standalone main isolation and its independent source checks are in progress; +the integrated result does not claim a main-only benchmark. + +The actual before-fix Beginner observation on frozen4904884 is execution +6.981277943s, total9.296563831s, compilation1.451889992s. It intentionally +preserves the reproduced issue487 metric bug and missing63parentmeans; no +parity exemption or speedup is claimed. Controller/source/environment/native +and complete actual input freezes pass. Source-freeze SHA256: +dc8a40519a10b7dd13ba9809a95bd6e2070622a326afa66ee2684adbe892a28d. +Issue487's source fix passes156 controls and original scoped R0/R1, but actual +candidate pipeline science and paired ordinary timing remain outstanding. From 7b8461bd1790b10cba90d6be83cb2013ed4b73ab Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 18:49:50 -0400 Subject: [PATCH 096/125] docs(perf): record cached metric gains and remaining contract gaps --- .../shared_runtime_plumbing_384_20261001.rst | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 29d47c301a..ef134f39ad 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1879,3 +1879,76 @@ and complete actual input freezes pass. Source-freeze SHA256: dc8a40519a10b7dd13ba9809a95bd6e2070622a326afa66ee2684adbe892a28d. Issue487's source fix passes156 controls and original scoped R0/R1, but actual candidate pipeline science and paired ordinary timing remain outstanding. + +Metric policy, saved parity and readiness follow-up, 2026-10-02 +------------------------------------------------------------- + +Main3705c071b13c29d354417d85dccf78c765b4a580 is normally merged at05ea538a. +Independent PR488 closes480 and PR489 closes487. Their production changes +are already integrated; subsequent main synchronization changes documentation +only. PR489 passes156 controls and original scoped R0/R1. Existing metric +declarations now derive enabled output columns; disabled Costes computations +and columns are omitted without changing enabled numerical implementations. + +Frozen baseline4904884 and sealed candidate d567d4d7 each have two cached +ordinary observations. Baseline execution5.527808428/5.327760458s and total +7.723725582/7.677667536s compare with candidate execution4.583703041/ +4.648376942s and total6.814110309/6.839743058s. Means improve execution +5.427784443 to4.616039992s (0.811744451s) and total7.700696559 to6.826926684s +(0.873769875s). Startup/readiness and shutdown are excluded; one worker +executes inline, with ordinary OUTCOMES and memory observation enabled. +This is a full-branch cached comparison, not an exclusive Costes attribution, +ABBA experiment, statistical claim or fully prewarmed qualification. + +The original baseline6.981277943/9.296563831s is excluded from that comparison: +three kernel overloads were written inside its pipeline timing window. The +later runs have no cache writes, which does not prove absence of cache loads. +Existing public library preparation omits object correlation and prepares +float64 threshold/RWC arrays; canonical runtime stages require float32 arrays. +A fresh private-cache public-preparation probe refuses later compilation and +cache loading: base passes; correlation, threshold and RWC fail. Requested +argument types exactly match retained production cache signatures. This is +readiness evidence using lawful synthetic values, not actual-input science +or performance evidence. Issue491 tracks canonical-stage preparation. + +All seven common CSV domains and two images are unchanged;100 unauthorized +Costes columns disappear (80 canonical and20 derived parent columns). +Candidate repeat scientific bytes are exact. Complete saved native replay +still fails solely for63 missing parent-mean features in both repetitions, +with no Costes, image, other numeric or physical-inventory differences. All +five relationship correlation keys and2093 pairs agree. Numerical tolerances +and required outputs are unchanged; full-case native parity remains RED. + +Issue492 identifies its first loss: all16 object-intensity tables exist, and +the producer has channel groups1/2/5/3, but compilation selects only child +channel3 for the prior-measurement input. Other channels never reach runtime +scope filtering. Repair belongs to the existing artifact relation/projection +contracts and must preserve child subject and image-set correlation. It adds +required scientific work; no speedup is claimed for this correctness repair. + +The bounded joint diagnostic pipeline succeeds with exact scientific bytes, +but capture admission is RED: no batch arrays are captured, and one final +store metadata event exceeds the4MiB cap. Earlier complete selection events +establish the bounded first-loss fact. The worker profile reaches28 serial +measurement invocations and zero batch calls; the earlier assumed batch +capture route is invalidated. A narrower serial-owner capture and nominal +batch-executor transport audit are next, not a claimed admitted replay. + +Cached comparison receipt: +/var/tmp/openhcs-beginner-metric-487-ordinary-cached-comparison-v1-20261002.json +SHA25636883d131c224f20b44139b02078c6ddba8b78a2e598901651910e12a92e4c2b. +Strict saved native receipt: +/var/tmp/openhcs-beginner-metric-487-strict-native-science-v1-20261002/receipt.json. +Fresh READY probe: +/var/tmp/openhcs-colocalization-public-ready-signatures-v2-20261002/observations.json. +Parent-mean first-loss receipt: +/var/tmp/openhcs-beginner-parent-means-first-loss-v2-20261002.json +SHA256c127ff3c79d82ddad4df38bb197606a25ee44a934e1909759922fc62b560f135. +Joint diagnostic admission/science receipt: +/var/tmp/issue487-joint-v2-original-admission-and-science-20261002.json +SHA256b29d5df0cd203e40a6e7f018e59b42d91e815a21af3e623811bc6b87a00ae53e. + +Whole-branch scalar-classifier R0 and issue479 validator-relocation R1 remain +RED. Installed419/433/435/450, original native image differences, full30-case +and scaling reruns and fresh figures remain open. The optimization goal stays +active; these measured gains do not redefine or complete the target. From 4c9cd972895c2f85a32d65dfa457ae3f81f3bdfd Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 19:20:16 -0400 Subject: [PATCH 097/125] docs: claim bounded live fractional-Z point projection repair (refs #152) --- ...live-fractional-z-point-projection-152.rst | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 docs/validation/live-fractional-z-point-projection-152.rst diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst new file mode 100644 index 0000000000..809b0af302 --- /dev/null +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -0,0 +1,37 @@ +Live fractional-Z point projection (partial152) +============================================== + +Owner: Planck. Parent retains152/134 continuous source-bearing native acceptance. +No competing production claim found:468 changes tracking.py;404 carries saved +writer proposals/evidence;394 does not change Napari/viewer sources. Shared +materialization source ownership must be coordinated before editing that seam. + +Reported technical failure +-------------------------- + +Execution5a4d513e-8f27-463f-8177-c657c5457f44 retained failure during viewer +settlement after persisted measurement/point outputs. Read only native6015 log +lines63..114: display_layer_batch -> display_axis_projection -> existing +NapariPointsLayerDisplayHandler.geometric_component_values rejects: +"Fractional-Z point ROI requires a projected z_index axis." +No scientific images, measurement rows, golden answers or pipeline parameters +were read. No endpoint was contacted and no scientist was sent hints. + +Investigation/acceptance +------------------------ + +Trace original typed PointROIOptions writer, ROIFractionalZ, source-plane +provenance, artifact source emission, live component semantics, and persisted +reopening as one family. Generate an independent tiny source-bearing PURE_3D +fixture; identify configuration/layout misuse versus lost Z domain. Preserve +existing absent-axis/domain guards. No mirrored axis domain, alternate codec, +ad-hoc registration or fabricated source anchor is acceptable. + +AST owner+all-consumer inventory precedes changes. Existing refactor-audit +Package/ParsedModule loader is reused. Scope <=512MiB/one CPU/noSwap,60s/shard; +no native/viewer/catalog/display startup or science. Source checkpoints and +proportionate controls will be recorded here. Installed real-path acceptance +requires a later parent-released terminal94/95/96 slot and is NOT claimed. + +This partial repair must not close152's remaining native orthogonal acceptance +or134's saved-output ownership. Current phase03 installed target stays immutable. From e742defc2e57d576362a2c9115e1ca6429a3847f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 19:34:29 -0400 Subject: [PATCH 098/125] docs(perf): record current generic-query frontier and acceptance limits --- .../shared_runtime_plumbing_384_20261001.rst | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index ef134f39ad..043b0a6d65 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1952,3 +1952,80 @@ Whole-branch scalar-classifier R0 and issue479 validator-relocation R1 remain RED. Installed419/433/435/450, original native image differences, full30-case and scaling reruns and fresh figures remain open. The optimization goal stays active; these measured gains do not redefine or complete the target. + +Canonical READY and current generic-query frontier +------------------------------------------------- + +PR493 is merged on mainb85f1e4add6075a71f4fd4e5ba7e2512e48990e5 and +closes491. Root normally integrates it atb015526c7c491abf80ea0958a08eb8723131c152. +Canonical image/object stages replace the duplicate kernel list and threshold +arithmetic.23 focused controls and original three-root R0/R1 pass; the actual +root fresh-cache integration control passes1test/9.86s. Compilation and disk +cache loading are refused after public preparation across IMAGE/OBJECT/BOTH +and threshold/Costes option states. This establishes readiness, not a speedup. + +One source-pinned Beginner diagnostic on clean b015 completes with exactly one +existing worker profile, nonempty existing runtime events,108 actual CP +executor calls,133 declared callable calls and no diagnostic errors. Profile +and event files occupy1.19MiB. Ten prelaunch controls preserve original calls, +errors, alias/mutation, descriptor/loader and thread partition behavior. The +instrumented12.9997s execution/15.2588s total is not an ordinary benchmark or +a regression. Profiling overhead is substantial and nonuniform. + +The same-thread orchestrator region partitions exactly12.988882s. This region +omits prior source/config decode; it is not the complete server-job clock. +Disjoint instrumented exclusive costs include module-output recording4.918576s, +declared callable invocations2.300491s, stack loading1.007764s, per-object output +recording0.920284s and worker residual0.861094s. cProfile costs overlap these +regions and must not be added or scaled to ordinary clocks. + +The determining generic query path makes312 full-column queries directly from +upstream parent-mean derivation:803488 structural-missing checks and195808 +qualification calls within those queries. Whole-worker structural checks are +1187866. Qualification precedes feature/source/row/object selection. Issue496 +tracks an operation-local schema/index fold on the existing nominal owners. +Opaque callback order, errors and mutation must remain observable; explicit +NaN propagation differs from default finite-value qualification. No new cache, +callback-identity dispatch or per-Relate numerical optimization is admitted. + +Strict replay of this diagnostic retains exactly63 missing parent means in both +saved native repetitions, with zero other numeric, image or inventory failures. +All seven CSVs/two TIFFs are byte-identical to ordinary candidated567; full +physical inventories agree and five correlation keys/2093pairs match. Source, +outputs, inputs, native and dependencies remain unchanged through replay. +Final all-zero science assertion correctly fails: no63-feature waiver exists. +Receipt /var/tmp/openhcs-current-generic-job-profile-strict-native-science-v1-20261002/receipt.json +SHA256bbc135e6de97628ddbabe8d350381a76a8af3000b4fba28f19e1879c27a220a8. + +Saved CSV readers cannot reconstruct preexport long rows, structural padding, +explicit missing cells, ownership or query order. That route is rejected; +metadata/relationship reader-domain errors are harness counterevidence, not +science failures. A bounded post492 actual logical-column/query capture must +demonstrate material absolute payoff before production optimization. The prior +ordinary Relate span near0.8s is only a whole-operation ceiling; the corrected +four-channel frontier and transfer remain unmeasured. + +Independent client residual evidence is0.818/0.804s for cached Beginner and +0.379s for QC. Two submissions occupy0.367/0.368s and0.120s respectively; +waiting beyond server clocks occupies0.03-0.08s. Four pipeline renders and +three config renders are observed in source, but their exact elapsed and byte +equality remain unproved. Blind memoization is rejected because shallow-frozen +documents contain mutable steps. This client-only route cannot remove server +execution work or independently close the seconds-wide target gap. + +Issue492's prepared four-channel/two-site repair passes its original controls +and guards. New prerequisite495 proves complete dynamic queries previously +ignored compiled producer path/backend, allowing foreign locations or an +ambiguous retained producer after legal rebinding. Existing planned query and +location authorities own the repair and point-in-time address law. Combined +qualification and actual full native63-field acceptance remain pending. + +The shared /home filesystem temporarily blocked Git commits. Only the owned +mutable worktree's website projection was made sparse through standard Git; +all75 website files were first preserved byte-exact at +/var/tmp/openhcs-owned-runtime-website-sparse-preserved-20261002/website with +their manifest receipt. Git objects retain the same files. This freed27MiB; +canonical ROOT, source code, benchmark inputs/outputs, dependencies and shared +caches were untouched. Earlier b015 source freezes completed before this +projection change; future freezes must explicitly record the sparse website. +All remaining acceptance and full-catalog/scaling/figure obligations stay open. From 8c1e3160d5c15a5b70197679883703f360f2d22e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 19:43:49 -0400 Subject: [PATCH 099/125] Document original source-bearing point-stream reproducer and ownership boundary --- ...live-fractional-z-point-projection-152.rst | 122 ++++++++++++++++++ .../synthetic_live_fractional_z_points.py | 106 +++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 docs/validation/synthetic_live_fractional_z_points.py diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 809b0af302..660ac705f1 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -35,3 +35,125 @@ requires a later parent-released terminal94/95/96 slot and is NOT claimed. This partial repair must not close152's remaining native orthogonal acceptance or134's saved-output ownership. Current phase03 installed target stays immutable. + +Source-only determining checkpoint +---------------------------------- + +Production base:1a89e09ebbf2bb744d5104cf6f70de93119f1adf. Ownership branch +fix/live-fractional-z-point-projection-20261002; first checkpoint4c9cd972895c. +Ten determining owner/consumer files compare byte-identical to the readonly +engineering484 ordinary installed target: materialization core/options, +function_artifact_materialization, stream_component_semantics, +roi_point_metadata, roi_source_metadata, source_image_provenance, +viewer_streaming_service, napari_viewer_server and napari_streaming_handlers. +This is not a claim of equality to all current main files. + +The independent executable reproducer is +``docs/validation/synthetic_live_fractional_z_points.py``. It creates only +four own5x7 uint16 source TIFFs (4x5x7 volume), invokes an original declared +PURE_3D NumPy producer, computes a centroid atZ1.5/Y1.5/X2.5, and materializes +its typed MeasurementTable through the original PointROIOptions disk writer. +Raw fixture spacing is2/.65/.65; the archive writer carries original source +provenance, not a newly fabricated physical calibration. No scientific pixels +or measurement rows are consumed. No compilation session, runtime process, +catalog, viewer instance, Qt application or endpoint connection is started. +The typed unused endpoint6199 is only a StreamingViewerSurface declaration. + +Actual original-path observations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Saved archive ROIFractionalZ domain is exactlyZ0,1,2,3, with original paths + and component metadata. Its common scalar address omits varyingZ. +* MaterializationSpec.emitted_source_identities returns no identities for + PointROIOptions. This method describes separately emitted projected planes, + not all source planes represented by a single point archive. +* ArtifactStreamSourceMetadataAuthority.payload_source_identity calls + image_payload_metadata. MeasurementTable owns SourceImageProvenanceFields, + not ImagePayloadMetadataCarrier, so absent context fallback this query + returns empty metadata. No-axis live scope results. A valid common scalar + fallback retains other axes but still excludesZ. +* StreamComponentMessageExtraAuthority scopes component order from those + incomplete items before constructing the domain. The unmodified Points + geometric_component_values rejects both scopes with the exact reported + missing projected-Z error. +* An exact first-source-plane fallback restores only singletonZ0. The same + guard rejects Z1.5 as exceeding that domain. An anchor alone is not a repair. +* Supplying the complete saved source-domain through the existing typed + StreamSourceComponentMetadataItems/from_viewer_surface projection succeeds: + the unmodified original Points guard reports occupiedZ0,1,2. This exercises + the same domain projection used by persisted reopening; it is NOT a native + reopen or live settlement acceptance journey. + +Thus the source-only fixture proves a live producer-to-projection contract gap, +not a need to remove the Points guard. No inference is made about the scientist's +algorithm, detected objects or expected counts. + +Reproduction and bounds +~~~~~~~~~~~~~~~~~~~~~~~ + +The installed control ran in a systemd user scope with MemoryMax512M, +MemorySwapMax0, CPUQuota100%, affinityCPU3 and timeout60s; terminal0. Use the +existing paired Python and readonly engineering484 import root:: + + systemd-run --user --scope --unit=openhcs-points152-final-control07 \ + -p MemoryMax=512M -p MemorySwapMax=0 -p CPUQuota=100% \ + timeout 60s taskset -c 3 env \ + PYTHONPATH=/home/ts/wt/openhcs-issue-batch-20260929/engineering484/installed \ + OPENHCS_CPU_ONLY=true OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 \ + /home/ts/wt/openhcs-paired-raw-installed-parent-20261001/.venv/bin/python -B \ + docs/validation/synthetic_live_fractional_z_points.py \ + --output /home/ts/wt/openhcs-issue-batch-20260929/live-points-152-synthetic07 + +The output directory must not exist; select a distinct owned path for a new +engineering invocation. Original07 artifacts are retained and not overwritten. +Terminal PTY62122, invocationdb70c70201d64334835bd7d02aa6ff47. +Source SHA256a22b227bce171597944785f282f8530fdc8ecc415068d42f5522fbc4ca3a9896. +Archive SHA2568276d6ad1d0ce1444aa17c78b367af9b929b6a6187163c913451c4bcb7b58b09. + +AST and honest failures +~~~~~~~~~~~~~~~~~~~~~~ + +Original refactor-audit ParsedModule/Repository traversal inventories704 +OpenHCS modules plus95 actual paired PolyStore/ZMQRuntime modules,364 named +owner/consumer sites, no parse omissions. Persistent evidence: +``/home/ts/wt/openhcs-issue-batch-20260929/live-points-152-AST-BEFORE03.json``. +This is named-family source evidence, not a complete NRA detector proof or +dynamic resolution/equivalence proof. Retained all-at-once first inventory was +OOM-killed at512MiB; second path-loader trial covered0 OpenHCS modules and was +not accepted. The file-at-a-time third inventory supplies the stated coverage. + +Earlier synthetic harness failures are retained separately:01 fake non-dataclass +surface;02 unsupported MeasurementTable source_voxel_spacing keyword; +03 str rather than Path for original ZIP loader; observation03 mappingproxy +passed to bare JSON rather than existing to_jsonable. These are not product +failures or successful reproductions. Observation04, guard05, complete06 and +complete07 controls terminated0. No production source was changed to pass them. + +Owner decision and remaining closure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +NRA/refactor-audit patterns IDEN-1/2 distinguish represented source span from +number of emitted files; BOUND-2 forbids bypassing the provenance owner; +IMPL-4/12 require an actual declaration-owned implementation used by the live +consumer rather than a new viewer-local fix or duplicate domain builder. + +Root394 head e742defc2e57d576362a2c9115e1ca6429a3847f owns shared materialization +core and function_artifact_materialization/source plumbing. Parent explicitly +holds those files pending owner release. No production edit is made here. +The exact proposed seam is posted on394 (comment5963068617), but affirmative +release is not yet recorded at this checkpoint. + +Proposed bounded closure: PointROIOptions declares represented stream source +identities through the existing options/spec family; its point writer preserves +the full original archive metadata and supplies an address derived from the +exact first provenance plane. Migrate the two original emission-query consumers +to the truthful stream-domain contract, retaining other writer behavior. Never +pretend PointROIOptions emits multiple plane files; never set a syntheticZ +on common metadata, add a parallel source-span store or weaken native guards. +Root may implement that narrow producer seam instead and coordinate its live +consumer; Planck owns this494 investigation and regression fixture. + +Before acceptance: coordinated coherent producer/consumer patch, original +2D/3D/multiple-artifact/bad-domain controls, after-inventory deletion evidence, +and parent-released installed continuous source-bearing live/reopen native +journey. No source fix, scientist rerun or native acceptance is claimed yet. diff --git a/docs/validation/synthetic_live_fractional_z_points.py b/docs/validation/synthetic_live_fractional_z_points.py new file mode 100644 index 0000000000..458a521aad --- /dev/null +++ b/docs/validation/synthetic_live_fractional_z_points.py @@ -0,0 +1,106 @@ +"""Tiny original PURE_3D producer -> typed Points writer -> live-domain control.""" +from pathlib import Path +import argparse +from openhcs.serialization.json import to_jsonable +import json +import numpy as np +import tifffile + +from openhcs.core.config import NapariStreamingConfig +from openhcs.core.memory import numpy as numpy_func +from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.core.measurement_row_materialization import MeasurementSparseColumnarRows +from openhcs.core.runtime_measurements import MeasurementTable, MeasurementSubject, MeasurementScope, ObjectCoreMeasurementFeature +from openhcs.core.runtime_tabular_values import FieldSpec +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.processing.materialization import MaterializationSpec, PointROIOptions, materialize +from openhcs.core.steps.function_artifact_materialization import ArtifactStreamSourceMetadataAuthority +from openhcs.core.steps.stream_component_semantics import StreamComponentMessageExtraAuthority, StreamSourceComponentMetadataItems +from openhcs.core.streaming_config_factory import StreamingViewerSurface, StreamingViewerRuntimeConfig +from openhcs.core.streaming_config_declarations import ViewerType +from polystore.streaming.viewer_transport import ViewerStreamSourceIdentity +from polystore.filemanager import FileManager +from polystore.disk import DiskStorageBackend +from polystore.roi import load_rois_from_zip +from openhcs.core.roi_point_metadata import ROIFractionalZ +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from zmqruntime.viewer_protocol import ViewerTransportEndpoint +from zmqruntime.config import TransportMode + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output', type=Path, required=True, help='New owned persistent synthetic directory; must not exist') +root = parser.parse_args().output +root.mkdir(exist_ok=False) +components = tuple({'well':'A01', 'site':1, 'channel':1, 'z_index':z, 'timepoint':1} for z in range(4)) +paths = tuple(str(root / f'A01_s001_w1_z{z:03d}_t001.tif') for z in range(4)) +volume = np.zeros((4,5,7), dtype=np.uint16) +volume[1:3,1:3,2:4] = 1 +for z, path in enumerate(paths): + tifffile.imwrite(path, volume[z]) +provenance = SourceImageProvenancePlanes.from_components(paths=paths, component_metadata=components) +metadata = ImagePayloadMetadata(source_image_provenance_planes=provenance, source_component_metadata={k:v for k,v in components[0].items() if k!='z_index'}, source_voxel_spacing=SourceVoxelSpacing((2.0,0.65,0.65)),plane_axis=RuntimePlaneAxis.SOURCE_BINDING) + +@numpy_func(contract=ProcessingContract.PURE_3D) +def synthetic_centres(image: np.ndarray) -> MeasurementTable: + assert image.shape == (4,5,7) + z,y,x = np.argwhere(image > 0).mean(axis=0) + return MeasurementTable(name='synthetic_centres', + rows=MeasurementSparseColumnarRows.from_rows(({'object_label':7,'center_z':float(z),'center_y':float(y),'center_x':float(x)},), + fields=(FieldSpec('object_label',int),FieldSpec('center_z',float),FieldSpec('center_y',float),FieldSpec('center_x',float))), + source_image_provenance_planes=provenance, source_component_metadata=metadata.source_component_metadata, + subject=MeasurementSubject(MeasurementScope.OBJECT,'synthetic','object_label')) + +table = synthetic_centres(metadata.attach_to(volume)) +spec = MaterializationSpec(PointROIOptions(z_feature=ObjectCoreMeasurementFeature.CENTER_Z,y_feature=ObjectCoreMeasurementFeature.CENTER_Y,x_feature=ObjectCoreMeasurementFeature.CENTER_X)) +archive = materialize(spec,data=table,path=str(root/'centres'),filemanager=FileManager({'disk':DiskStorageBackend()}),backends=['disk'],backend_kwargs={}) +items = ArtifactStreamSourceMetadataAuthority.metadata_items(materialization_spec=spec,data=table,fallback_source_identity=None) +surface = StreamingViewerSurface( + runtime_config=StreamingViewerRuntimeConfig(transport_endpoint=ViewerTransportEndpoint(host='127.0.0.1',port=6199,transport_mode=TransportMode.TCP),persistent=True,viewer_type=ViewerType.NAPARI), + display_config=NapariStreamingConfig(enabled=True), source=ViewerStreamSourceIdentity(microscope_handler=None,plate_path=None)) +authority = StreamComponentMessageExtraAuthority.from_viewer_surface(surface,source_metadata_items=items) +saved_rois = load_rois_from_zip(Path(archive)) +saved_metadata = ROIArchiveSourceMetadata.decode(saved_rois) +saved_domain = ROIFractionalZ.source_component_domain(saved_rois, saved_metadata) +reopened = StreamComponentMessageExtraAuthority.from_viewer_surface(surface, source_metadata_items=StreamSourceComponentMetadataItems.from_values(saved_domain)) +result = dict(saved_domain=saved_domain, archive_scalar_metadata=saved_metadata.source_component_metadata, archive=archive,shape=volume.shape,rows=tuple(table.iter_row_mappings()),source_component_order=authority.layout.component_order,declared_domain=authority.payload().to_wire_mapping(), scientific_data_opened=False,runtime_started=False) +print(json.dumps(to_jsonable(result),sort_keys=True)) +assert 'z_index' not in authority.layout.component_order, 'Expected original live-source defect no longer reproduces; update this engineering control' +metadata = saved_metadata +rois = saved_rois +live = authority +archive = Path(archive) + +from polystore.roi_converters import NapariROIConverter +from polystore.streaming.identity import StreamProducerIdentity +from polystore.streaming_constants import StreamingDataType +from openhcs.runtime.napari_streaming_handlers import NapariStreamLayerItem, NapariStreamLayerAddress +from openhcs.runtime.viewer_component_system import ViewerComponentValueDomainPayload +from openhcs.runtime.napari_viewer_server import NapariPointsLayerDisplayHandler + +anchor = metadata.source_provenance.for_source_plane(0).scalar_source_identity +domain = ViewerComponentValueDomainPayload.from_wire_mapping(reopened.payload().to_wire_mapping()['component_value_domain'],context='own synthetic source-bearing points') +item = NapariStreamLayerItem( + data=NapariROIConverter.rois_to_shapes(rois), + producer=StreamProducerIdentity.pipeline_output(output_kind='artifact',output_key='synthetic_centres',projection_key='synthetic_centres',step_name='Own engineering only',pipeline_position=0,artifact_kind='roi'), + address=NapariStreamLayerAddress(components=anchor.component_metadata,path=str(archive),stream_layer_data_type=StreamingDataType.POINTS), + image_metadata=metadata,plane_component_domain=domain) +handler = NapariPointsLayerDisplayHandler() +results = {} +for name, fallback in [('none',None),('common',metadata.source_provenance.scalar_source_identity),('first_plane',anchor)]: + items = ArtifactStreamSourceMetadataAuthority.metadata_items(materialization_spec=spec,data=table,fallback_source_identity=fallback) + authority = StreamComponentMessageExtraAuthority.from_viewer_surface(surface,source_metadata_items=items) + try: + geometry = handler.geometric_component_values([item],authority.component_axis_semantics) + except ValueError as error: + results[name] = dict(order=authority.layout.component_order,error=str(error)) + else: + raise AssertionError(f'Incomplete live domain unexpectedly accepted: {name}: {geometry!r}') +positive = handler.geometric_component_values([item],reopened.component_axis_semantics) +assert results['none']['error'] == 'Fractional-Z point ROI requires a projected z_index axis.' +assert results['common']['error'] == results['none']['error'] +assert 'exceeds the declared Z domain' in results['first_plane']['error'] +assert positive == {'z_index':[0,1,2]} +print(json.dumps(dict(negative=results,original_domain_positive=positive,viewer_started=False,connected=False),sort_keys=True)) From 957a03f8b21e68639f9de6fd85f0738e319dd179 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 19:52:46 -0400 Subject: [PATCH 100/125] Record Root394 point-stream integration adjustment at pinned head --- ...live-fractional-z-point-projection-152.rst | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 660ac705f1..1e00151a96 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -157,3 +157,43 @@ Before acceptance: coordinated coherent producer/consumer patch, original 2D/3D/multiple-artifact/bad-domain controls, after-inventory deletion evidence, and parent-released installed continuous source-bearing live/reopen native journey. No source fix, scientist rerun or native acceptance is claimed yet. + +Root394 integration review at4110503 +------------------------------------ + +Reviewed the pinned current Root394 head +4110503cb543ed525d787f678abdf6efe374b8b7 against production base1a89e09e. +Of the ten determining files, only source_image_provenance.py, +function_artifact_materialization.py and materialization/core.py changed. +The seven remaining files, including PointROIOptions, the Points guard, +ROIFractionalZ, saved reopening and component scoping, are unchanged. +This is a source diff/semantic review, not a new scan or execution claim. + +The required integration adjustment is to use Root's MaterializationBatch +render/save and SavedMaterializationOutputs ownership. Keep the exact typed +point Output and its complete archived provenance flowing through that batch; +do not restore deleted output-group rendering or re-derive publication paths +from logical records. MaterializedRuntimeArtifact now observes actual successful +backend outputs, so the point repair must not add a competing save/path ledger. + +SourceImageIdentity and common metadata now delegate snapshots/composition to +SourceMetadataFields, preserving OwnedSourceMetadataFields lifetime. Derive +represented identities and the anchor through existing provenance methods, +without flattening those records to a separate raw mapping. The source-provenance +initializer now accepts the raw InitVar tuple and decodes at its owner; +MeasurementTable's consumer is migrated accordingly. The published fixture uses +the original constructor, not a direct call to that internal initializer. + +The remaining defect seam is unchanged: artifact backend kwargs are still built +before prepare_materialization renders its batch, using the same +ArtifactStreamSourceMetadataAuthority.metadata_items and +MaterializationSpec.emitted_source_identities logic. Root's new save ledger does +not provide a represented PointROI Z domain to that earlier live projection. +Both full source span and exact original anchor obligations remain. + +No production edits, tests, scans, installs or endpoint requests were performed +for this review. The original8c1e3160d fixture/control remains published and +qualified only at its original immutable installed484 strength, not executed +against Root4110503. Independent consumer diagnosis is complete. Named dependency: +Root394's affirmative shared-seam release or coordinated producer implementation. +The hold and parent-owned later installed native acceptance remain in force. From 5d50a0bb46ab4ccdd17bb9924ce698893f5ed85b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 20:10:15 -0400 Subject: [PATCH 101/125] docs(perf): qualify corrected producer domain and native parity --- .../shared_runtime_plumbing_384_20261001.rst | 97 ++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index 043b0a6d65..f928de0fba 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -1985,7 +1985,7 @@ qualification calls within those queries. Whole-worker structural checks are 1187866. Qualification precedes feature/source/row/object selection. Issue496 tracks an operation-local schema/index fold on the existing nominal owners. Opaque callback order, errors and mutation must remain observable; explicit -NaN propagation differs from default finite-value qualification. No new cache, +NaN propagation differs from default measurement-presence qualification. No new cache, callback-identity dispatch or per-Relate numerical optimization is admitted. Strict replay of this diagnostic retains exactly63 missing parent means in both @@ -2029,3 +2029,98 @@ canonical ROOT, source code, benchmark inputs/outputs, dependencies and shared caches were untouched. Earlier b015 source freezes completed before this projection change; future freezes must explicitly record the sparse website. All remaining acceptance and full-catalog/scaling/figure obligations stay open. + +Corrected producer domain, native parity and residual measurement frontier +------------------------------------------------------------------------ + +PR497 merges on main49f2fd3639363b7cc5e716eb3ba3a7358aa85429 and formally +closes492 and495. Its independently qualified production commit iseeed09233; +final6da19179 adds tracked qualification only. Original unchanged R0/R1 pass +against normally merged main6ce527a29 with no increased finding and unchanged +R1 budget.211 related controls pass;212 integrated controls pass11.78s with +two unrelated plugin configuration warnings. Global supplemental ownership +census covers704 modules/5228 original classes with no AST errors. + +InputObjectMeasurementSourceRelation derives complete measurement-producer +consumption through the existing input projection owner. The old planner and +RelateObjects selection algorithms are removed. Ordinary input and PLATE +selection remain intact. Dynamic queries resolve the actual compiled producer +path/backend and snapshot mutable group-address maps at query birth. The +meaningful old store-pickle control passed before planned lookup but failed +after caching a mapping-proxy address. Both lookup domains now use the existing +bounded store-owned cache lifetime; obsolete flat cache fields, initialization, +invalidation and transport paths are deleted. Ordered records, empty results, +replacement revision, eviction, primary/current/observation state and aliases +are covered. Arbitrary external plan/mapping callbacks and query-target callback +counts under bounded eviction remain outside the concrete production audit. + +Root integrates production at5ed0d13978db1cda92a490614d326525ad552af7. A fresh +uninstrumented ordinary Beginner1w1t completes with execution4.650205374s, +compilation1.603918076s and total7.070084621s. Mandatory READY, registry/kernel +warmup, process startup and shutdown are excluded. Execution is inline with one +thread; no measured fork/spawn startup. Default OUTCOMES and ordinary memory +observation are retained. The tracked75-file website-only sparse projection is +explicitly inventoried; no production source/input/dependency is omitted. +Source, Python, installed dependencies, native binaries and exact image/CPPipe +hashes are fixed before and after. This single correctness observation is not +causal optimization or statistical evidence. + +The unchanged strict reader consumes all seven CSVs and two TIFFs against BOTH +retained native repetitions: ZERO numeric/measurement differences, ZERO image +differences, complete physical inventory, and five correlation keys/2093pairs +on both sides. The63 missing parent means are restored. Only MyExpt_Nuclei.csv +changes scientifically versus the previous ordinary result. Reader/source and +output hashes remain unchanged through comparison. This execution uses the +integrated394/479 reader: it qualifies the producer repair's outputs, not a +standalone main-only execution or whole394 architecture gate. +Receipt /var/tmp/openhcs-child-measurements-492-integrated-strict-native-science-v1-20261002/receipt.json +SHA25637930ee4622c5fc1e067fc853bef1ae627ebe74546c27c3cc65d9fb476343ba0. +Tracked independent qualification: +docs/validation/child-measurement-producer-selection-492-495-20261002/QUALIFICATION.rst. + +The retained native warm-process invocations are15.270141648s and14.634740414s, +mean14.952441031s. A descriptive comparison to this single corrected execution +is3.215x execution and2.115x including OpenHCS compilation/client completion; +these are different sample counts, not a causal change or statistical claim. +Native invocation includes prepare/modules/post-run/close, with import/JVM, +CPPipe load and its warmup excluded. Startup is not added to either pipeline. +Other cases and full-catalog/scaling figures are not qualified by this case. + +The next actual logical-query capture executes438 queries over seven tables, +not four times the previous312. Its processor succeeds and all nine scientific +files are byte-identical to the qualified5ed ordinary output; complete physical +inventories match. Diagnostic admission correctly fails before writing any +logical graph: unchanged V6 refuses the150000-node unique-instance inventory. +No partial graph, increased limit, saved-CSV substitution or performance claim +is admitted. The profiled/captured14.1409s execution is diagnostic overhead, +not an ordinary regression. Preserve the original failure and source freeze. +Artifact /var/tmp/openhcs-measurement-query-logical-capture-v1-20261002/observations.json. +Scientific-byte receipt: +/var/tmp/openhcs-measurement-query-logical-capture-v1-scientific-byte-comparison-20261002.json. + +A revised diagnostic representation may pack numeric answer-oracle leaves while +preserving every actual query and the complete determining input/schema/alias +graph. Node/category counts and original replay must establish admission before +any matched candidate timing or production change. Existing150000-node, +32MiB graph,4MiB ledger and512MiB free-space limits remain unchanged. Pure +callback declarations alone do not license skipping arbitrary scalar equality +or string effects; original opaque callback/error/mutation order remains a +separate production wiring gate. + +The output-recording ownership audit also rejects a misleading second frontier: +coarse recorder-exclusive4.9186s contains the query496 subtree. In cProfile, +record_module_outputs self time is0.000638s, and measurement-table construction +contains Relate augmentation4.1412s/upstream query3.3457s. Those diagnostic +times cannot be added. Independent100-call object recording is dominated by +split_scoped_rows0.5620s diagnostic. Its18735-row domain has80 colocalization, +16 concatenated and4 shape batches. Existing row owners could remove dictionary +expansion, but no credible multi-second ordinary payoff is established; the +standalone route is rejected pending a necessary shared-query dependency. +Audit /var/tmp/openhcs-b015-output-recording-ownership-audit-20261002.json +SHA256b0bf2c90f50c1da5022f6aa2a657779f0bdd9390c0adfb022a86b5db21b10779. + +Root normally merges current main after all5ed jobs complete, at899aaddce; +that merge changes documentation/evidence only relative to5ed production. +Whole394 scalar-classifier R0 and479 validator-relocation R1 remain RED. +Installed419/433/435/450, outstanding native image differences, full30/scaling +reruns and fresh figures remain open. The unlimited optimization goal is ACTIVE. From 2d3d75e80309472f6b6b26af067da6e69aa32ab6 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 20:25:20 -0400 Subject: [PATCH 102/125] docs(perf): separate plate export from callable timing --- .../shared_runtime_plumbing_384_20261001.rst | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/validation/shared_runtime_plumbing_384_20261001.rst b/docs/validation/shared_runtime_plumbing_384_20261001.rst index f928de0fba..ef92751224 100644 --- a/docs/validation/shared_runtime_plumbing_384_20261001.rst +++ b/docs/validation/shared_runtime_plumbing_384_20261001.rst @@ -2124,3 +2124,55 @@ that merge changes documentation/evidence only relative to5ed production. Whole394 scalar-classifier R0 and479 validator-relocation R1 remain RED. Installed419/433/435/450, outstanding native image differences, full30/scaling reruns and fresh figures remain open. The unlimited optimization goal is ACTIVE. + +Profiler-free exclusive ledger and plate export correction +--------------------------------------------------------- + +The source-pinned5d50a0bb46ab4ccdd17bb9924ce698893f5ed85b diagnostic completes +with BOTH cProfile/runtime profiler disabled,24 coarse nominal boundaries and +14 meaningful prelaunch controls. Source/environment/native/input and explicit +website-only sparse witnesses remain fixed before/after. It retains32,223bytes +of JSON, no profile/capture artifacts and no diagnostic errors. All nine +scientific output bytes and the complete11-file inventory are identical to the +strict-both-native-qualified5ed ordinary result. Managed metadata differences +are recorded separately. No new native execution or tolerance waiver is used. + +Public execution4.703251600s/compilation1.568818331s/total7.089778207s are +diagnostic, not a causal gain or an estimate of wrapper overhead. The execution +region4.692535908s partitions into23 nonzero owner groups:133 hooked callable +invocations1.923475937s and2.769059971s outside that boundary. The invocation +boundary includes metadata/filtering and does NOT cover all registered calls. +In particular, execute_plate_scoped_steps directly resolves/calls PLATE exports. +Actual ExportToSpreadsheet progress is0.707153320s; it lies inside the +server-exclusive0.734300637s bucket. These intervals overlap and cannot be +added or subtracted as exact disjoint method costs. Server reconstruction is +therefore not established as the dominant0.734s term. Source/config decoding +also precedes this region. Compiler-exclusive1.009028861s includes genuine +fresh compilation, not a second artifact-reuse compile. + +Current measured exclusive query cost is0.481040613s/438calls. Nonquery module +recording is0.405474477s (inclusive0.886515090s), object recording0.156714182s, +stack loading0.184871211s, CP contract execution0.212007483s, worker remainder +0.140013289s, CP module remainder0.130000117s and image request0.086237226s. +Saving/publication/unstacking individually are smaller. Recording remainder +contains actual table creation, parent means and other required processing; it +is not isolated provenance. All-query/all-module/all-object recording has an +impossible elimination ceiling1.043229273s before retaining mandatory work. + +This invalidates the single-query route as the main multi-second fix. Its +18-control compact numeric-oracle capture preparation is retained but will not +be frozen/launched/polished without a necessary broader dependency. Existing +V6 and original limits remain unchanged. The new investigation maps common +columnar table/schema derivation across queries, recorders and the actual +spreadsheet publication path, retaining opaque scalar/callback semantics. +The exporter needs its own authority/phase analysis before a payoff claim. + +Ledger /var/tmp/openhcs-current-generic-runtime-ledger-v2-20261002/observations.json. +Corrected analysis /var/tmp/openhcs-current-generic-runtime-ledger-v2-analysis-plate-corrected-20261002.json +SHA2568b74fb088fae2527563fb42d38f3dedf5ab5ed936c5de805cd394343e3d0529c. +Byte gate /var/tmp/openhcs-current-generic-runtime-ledger-v2-scientific-byte-gate-20261002.json. +Original first-classification and correction receipts are both retained. +PR394's current description is rewritten around the remaining implementation; +merged fixes are described as dependencies and closed issue clauses removed. +Open persisted-publication134 now has an explicit closing link to draft394, +alongside384/419/433/435/450/479/176/496. None is claimed complete merely linked. From 396379b06769a3f5afe92413b7d9d6a21dd3f66f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 21:45:57 -0400 Subject: [PATCH 103/125] docs: record current plumbing frontier and rejected performance routes Refs #496, #394. Preserve separate scientific, diagnostic and performance gates; record current ordinary Speckles and complete 3D plumbing breakdown. --- .../runtime_plumbing_frontier_20261003.rst | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 docs/plans/runtime_plumbing_frontier_20261003.rst diff --git a/docs/plans/runtime_plumbing_frontier_20261003.rst b/docs/plans/runtime_plumbing_frontier_20261003.rst new file mode 100644 index 0000000000..e34009a5cb --- /dev/null +++ b/docs/plans/runtime_plumbing_frontier_20261003.rst @@ -0,0 +1,134 @@ +Runtime plumbing frontier: current measurements and rejected routes +================================================================= + +This checkpoint supports PR #394 and issue #496. It records measurements and +remaining investigation, not a new production optimization. Main #498 was +normally merged and pushed at ``093c2f42d1235b5e218800f57937b6a52b4f5255``. +That transition changed documentation only; production sources remain identical +to the earlier native-qualified integrated source. + +Timing boundaries +----------------- + +Every run uses the original public benchmark driver, one worker and one CPU +thread, default OUTCOMES completion and the ordinary memory observer. Mandatory +registry/library/kernel preparation completes before server READY. Server +startup, preparation and shutdown are outside pipeline clocks. Execution excludes +compilation; total includes compilation and completion. Single-worker execution +is inline. The retained native invocation clock excludes import/JVM/pipeline +loading and includes prepare, modules, post-run and closure. + +Diagnostic nested exclusive clocks partition their original execution root. +They include observer overhead and required work. A callable boundary includes +its filtering, conversion and metadata behavior; it is not a pure-kernel clock. +Inclusive descendants cannot be added to their ancestor. Correct output bytes +do not establish performance improvement, input immutability or callback purity. + +Fresh ordinary Speckles +----------------------- + +Two independent uninstrumented invocations on the current source produced +execution times of 1.199828148s and 1.159908533s. Their means are: + +* Execution: 1.179868340s. +* Compilation: 1.067295671s. +* Total: 2.514483608s. + +Both complete five-file inventories, including all three scientific CSVs, +are byte identical to the qualified baseline. Physical images, pipeline, +configuration, server environment, dependencies and native evidence are joined +to the original receipts. Compared descriptively with the retained native +invocation mean of 1.915068373s, execution is 1.6231x faster and total is 0.7616x +as fast. The execution gap to 2x is 0.222334154s. Native was not rerun for this +checkpoint; two observations do not establish statistical significance or a +causal source improvement. + +The earlier and current pipelines both invoke IdentifyPrimaryObjects twice. +Their relevant source bodies and settings are identical. The prior slower +timing therefore has no established code-change explanation. The current +ordinary progress spans put about 0.820924s in those two steps, approximately +70 percent of execution. + +Actual declumping inputs and boolean output oracles were separately captured +under unchanged storage limits. The original two stage bodies took 0.070122s +and 0.001791s, totaling 0.071913s. Capture overhead was excluded from these +stage clocks. Even eliminating that stage cannot bridge the measured gap, so +the seed optimization route is stopped before a candidate or replay is built. +The original diagnostic wrapper import failure is retained separately; the +successful harness passed controls for all five actual installed hooks. + +Current 3D plumbing +------------------- + +An unchanged 24-boundary diagnostic on the current source completed with both +profilers disabled. Its execution root is 8.958889063s, partitioned into +4.163085915s at 211 callable boundaries and 4.795803148s outside them. The latter +is approximately 53.5 percent of execution. Selected disjoint exclusive terms: + +* Stack loading: 0.965303496s across 32 calls. +* Output saving: 0.608366542s across 28 calls. +* Worker-lane residual: 0.619469857s. +* Image-request construction: 0.390939402s across 30 calls. +* Module-output recording: 0.413856133s across 26 calls. +* Metadata publication: 0.493661132s across five calls. +* Artifact metadata reconciliation: 0.288939269s. +* Validation and unstacking: 0.189130950s across 28 calls. + +There are zero feature-query calls. The complete six-CSV/120-TIFF scientific +bytes match the qualified ordinary baseline, all 128 inventory paths match, +and three physical input volumes, the pipeline and retained native logical +volume/plane witnesses are verified. Only the managed metadata JSON differs, +with both hashes explicitly reported. This is a diagnostic breakdown, not an +accepted ordinary runtime gain. + +The prior ordinary 3D execution of 8.536s versus native 14.396s leaves roughly +1.34s to reach 2x execution. Publication and reconciliation together have only +a 0.7826s diagnostic ceiling before mandatory work. A publication-only change +is therefore insufficient. The investigation must span shared producer/source +derivation across loading, image requests, recording, saving and publication. +The existing nominal authorities are the intended owners; no extra mirror, +global cache, per-function shortcut or delayed per-step visibility is admitted. + +Measurement frontier and architectural constraints +-------------------------------------------------- + +The separate 42-boundary Beginner diagnostic measures a disjoint table +assembly/query/pivot envelope of 1.462782s. Native CSV rendering is another +0.163899s. All nine scientific output files and complete physical inventory +match the native-qualified baseline. Its occurrence graph reaches the original +4,096-event limit and remains RED independently of scientific correctness. +The complete Speckles graph demonstrates repeated access to one table and +object-label vector, but that entire measurement lane is only about 0.038s. +Neither query-only nor the 16-call intensity projection route can close the +dominant gaps, and the measurement fold is not a 3D optimization: that workload +does not perform these queries. + +Existing serialization views have distinct semantics. Raw projection source +metadata and enriched SOURCE_METADATA cannot share one output dictionary. +The transaction overlays raw path records before decoding surviving entries; +decoding overwritten malformed entries earlier would change first errors. +Serializer callbacks, independently mutable outputs, current per-step +publication visibility and concurrent-axis final pruning remain obligations. +A frozen projection declaration does not establish deep immutability of its +image metadata. Ownership proposals remain provisional until those laws and +representative replay show a sufficient end-to-end payoff. + +Retained evidence +----------------- + +The following local receipts contain exact source/controller/environment/input +pins, original commands, file inventories and hashes: + +* ``/var/tmp/openhcs-current-speckles-ordinary-pair-v1-20261003/observations.json``: + SHA-256 ``d08f35061285e8a4d543377c1de7993623654844eddcaea806d269672a0bf4a1``. +* ``/var/tmp/openhcs-speckles-seed-capture-v4-scientific-byte-gate-20261003.json``: + SHA-256 ``4da188db100b9526f18a6a02ce98872dad6e7ee5dca60c1b85e7052e1287e301``. +* ``/var/tmp/openhcs-current-3d-runtime-ledger-v1-scientific-byte-gate-20261003.json``: + SHA-256 ``36a4f2cdcc14baaa38f027a6dc759facd1229cc1f5960ddb063cbe689efda0cf``. +* ``/var/tmp/openhcs-measurement-occurrence-ledger-v1-scientific-byte-gate-v2-20261003.json``: + SHA-256 ``4352fa24e8984e0f4c67587ae674ff28085ed134531dea28fe7d5b8517671f99``. + +PR #394 remains draft. Whole-branch original R0 and the #479 per-file R1 +failures remain explicit and unwaived; installed consumer acceptance, +full-catalog/scaling measurements and fresh figures remain unfinished. +The performance goal remains active. From 3a11b695d6f586e33ce34b036afdf3d4df87ac9e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 22:33:40 -0400 Subject: [PATCH 104/125] fix: retain represented point source domain and exact live anchor --- ...live-fractional-z-point-projection-152.rst | 32 +++++++++++ .../function_artifact_materialization.py | 6 +- openhcs/processing/materialization/core.py | 47 ++++++---------- openhcs/processing/materialization/options.py | 55 ++++++++++++++++++- .../unit/test_file_materialization_options.py | 2 +- tests/unit/test_materialization_core.py | 2 +- 6 files changed, 109 insertions(+), 35 deletions(-) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 1e00151a96..574085c806 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -197,3 +197,35 @@ qualified only at its original immutable installed484 strength, not executed against Root4110503. Independent consumer diagnosis is complete. Named dependency: Root394's affirmative shared-seam release or coordinated producer implementation. The hold and parent-owned later installed native acceptance remain in force. + +Released producer implementation checkpoint +------------------------------------------- + +Root394 explicitly released the narrow options/spec declaration, typed point +output/writer anchor and original artifact source-query migration at +396379b06769a3f5afe92413b7d9d6a21dd3f66f. The source branch is normally merged +with main9531d594 (including504); the five foreign dirty gitlinks are untouched. +Source provenance construction, compiler and generic load/stack remain Root's +ownership and are not edited. No scientific catalog/endpoint is contacted. + +PointROIOptions now owns both typed measurement selection and the represented +stream source-plane domain. FileOutputOptions carries the previous projected +plane implementation; MaterializationSpec delegates rather than deciding which +writer qualifies. The misleading emitted_source_identities entrypoint is +deleted, with its artifact-metadata consumer and two test consumers migrated to +stream_source_identities. Number of files and represented source planes remain +distinct facts (IDEN-1/2, IMPL-4, BOUND-2). + +The original point writer returns PointROIOutput, an Output subtype whose +required original metadata retains the full archive provenance. Its stream +address derives from provenance.for_source_plane(0), without overwriting common +archive metadata or adding an anchor store. Existing Output replacement and +viewer backend batching consume that property normally. All fractional-Z +guards and persisted metadata formats remain unchanged. + +Fresh original refactor-audit file-at-a-time inventory is retained in +``live-points-152-owner-before09.json`` under the parent issue-batch root. +The preceding08 invocation rejected an invalid git revision WORKTREE before +reading source; it is not an accepted census. This is an implemented source +checkpoint awaiting bounded original-writer/projection regression verification; +it is not installed live settlement or native reopening acceptance. diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index 7d4a0da320..0432d45d35 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -454,10 +454,10 @@ def metadata_items( fallback_source_identity or ArtifactStreamSourceMetadataAuthority.payload_source_identity(data) ) - emitted_identities = materialization_spec.emitted_source_identities(data) - if emitted_identities: + stream_identities = materialization_spec.stream_source_identities(data) + if stream_identities: return StreamSourceComponentMetadataItems.from_source_identities( - emitted_identities, + stream_identities, fallback_source_identity=fallback_source_identity, ) return StreamSourceComponentMetadataItems.from_values( diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index faff29684a..fd4ba40656 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -344,6 +344,17 @@ def with_variable_components( ) +@dataclass(frozen=True, kw_only=True) +class PointROIOutput(Output): + """Full point-archive provenance with an exact source-plane stream anchor.""" + + metadata: ImagePayloadMetadata = field() + + @property + def source_identity(self) -> SourceImageIdentity: + return self.metadata.source_provenance.for_source_plane(0).scalar_source_identity + + @dataclass(frozen=True) class TextOutput(Output): """Materialization output whose canonical payload is text.""" @@ -3184,11 +3195,8 @@ def _write_point_roi_zip( from polystore.roi import PointShape, ROI from openhcs.core.roi_point_metadata import ROIFractionalZ from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata - from openhcs.core.runtime_measurements import MeasurementTable - payload = MaterializationInput.from_value(data, options).data - if not isinstance(payload, MeasurementTable): - raise TypeError("PointROIOptions requires a MeasurementTable payload.") + payload = options.measurement_payload(data) id_field = payload.subject.object_id_field if id_field is None: raise ValueError("PointROIOptions requires an object-measurement subject.") @@ -3259,7 +3267,7 @@ def _write_point_roi_zip( metadata = ImagePayloadMetadata(source_provenance=provenance) ROIFractionalZ.source_component_domain(rois, metadata) return [ - Output( + PointROIOutput( path=ctx.paths(options).primary_output_path(options), content=ROIArchiveSourceMetadata.bind(rois, metadata), metadata=metadata, @@ -4028,34 +4036,15 @@ def emits_variable_component_planes(self, data: MaterializationValue) -> bool: for output_options in self.outputs ) - def emitted_source_identities( + def stream_source_identities( self, data: MaterializationValue, ) -> tuple[SourceImageIdentity, ...]: - """Return source identities for writers that emit projected planes.""" + """Return the source domain represented by the selected writer stream.""" for output_options in self.outputs: - if not materialization_emits_variable_component_planes( - output_options, - data, - ): - continue - materialization_input = ( - MaterializationInput.from_runtime_slice_projected_value( - data, - output_options, - ) - ) - if len(materialization_input.items) == 1: - provenance = materialization_input.items[0].metadata.source_provenance - if provenance.source_plane_count > 1: - return tuple( - provenance.for_source_plane(index).scalar_source_identity - for index in range(provenance.source_plane_count) - ) - return tuple( - item.metadata.source_provenance.scalar_source_identity - for item in materialization_input.items - ) + identities = output_options.stream_source_identities(data) + if identities: + return identities return () def uses_filename_source_identity(self, data: MaterializationValue) -> bool: diff --git a/openhcs/processing/materialization/options.py b/openhcs/processing/materialization/options.py index d29b4e8f5f..8932702c60 100644 --- a/openhcs/processing/materialization/options.py +++ b/openhcs/processing/materialization/options.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional from openhcs.core.runtime_measurements import RuntimeMeasurementFeature from openhcs.processing.materialization.path_scopes import ( @@ -18,6 +18,11 @@ SharedMaterializationRelativePathScope, ) +if TYPE_CHECKING: + from openhcs.core.runtime_measurements import MeasurementTable + from openhcs.core.source_image_provenance import SourceImageIdentity + from openhcs.processing.materialization.core import MaterializationValue + class MaterializedFilenameIdentity(str, Enum): """Semantic identity used to construct materialized artifact filenames.""" @@ -53,6 +58,33 @@ def primary_output_suffix(self) -> str: """Suffix for this writer's primary materialized output.""" return self.filename_suffix + def stream_source_identities( + self, + data: MaterializationValue, + ) -> tuple[SourceImageIdentity, ...]: + """Return the source planes represented by this writer's stream.""" + from openhcs.processing.materialization.core import ( + MaterializationInput, + materialization_emits_variable_component_planes, + ) + + if not materialization_emits_variable_component_planes(self, data): + return () + materialization_input = MaterializationInput.from_runtime_slice_projected_value( + data, self + ) + if len(materialization_input.items) == 1: + provenance = materialization_input.items[0].metadata.source_provenance + if provenance.source_plane_count > 1: + return tuple( + provenance.for_source_plane(index).scalar_source_identity + for index in range(provenance.source_plane_count) + ) + return tuple( + item.metadata.source_provenance.scalar_source_identity + for item in materialization_input.items + ) + @dataclass(frozen=True) class SourceOptions: @@ -126,6 +158,27 @@ def __post_init__(self) -> None: if len({feature.measurement_row_field_name for feature in features}) != 3: raise ValueError("PointROIOptions coordinates require distinct row fields.") + def measurement_payload(self, data: MaterializationValue) -> MeasurementTable: + """Resolve the point writer's declared source to its typed measurements.""" + from openhcs.core.runtime_measurements import MeasurementTable + from openhcs.processing.materialization.core import MaterializationInput + + payload = MaterializationInput.from_value(data, self).data + if not isinstance(payload, MeasurementTable): + raise TypeError("PointROIOptions requires a MeasurementTable payload.") + return payload + + def stream_source_identities( + self, + data: MaterializationValue, + ) -> tuple[SourceImageIdentity, ...]: + """One point archive represents all original source planes, not one Z.""" + provenance = self.measurement_payload(data).source_provenance + return tuple( + provenance.for_source_plane(index).scalar_source_identity + for index in range(max(1, provenance.source_plane_count)) + ) + @dataclass(frozen=True) class SWCOptions(FileOutputOptions, SourceOptions): diff --git a/tests/unit/test_file_materialization_options.py b/tests/unit/test_file_materialization_options.py index d4ee275f12..2a97847618 100644 --- a/tests/unit/test_file_materialization_options.py +++ b/tests/unit/test_file_materialization_options.py @@ -85,7 +85,7 @@ def test_source_identity_image_file_projects_addressable_stack_planes() -> None: assert primary_path == "/analysis/site1_saved.npy" assert spec.emits_variable_component_planes(payload) assert tuple( - identity.path for identity in spec.emitted_source_identities(payload) + identity.path for identity in spec.stream_source_identities(payload) ) == tuple(f"/input/site{index}.tif" for index in range(1, 4)) for index, plane in enumerate(planes, start=1): np.testing.assert_array_equal( diff --git a/tests/unit/test_materialization_core.py b/tests/unit/test_materialization_core.py index 39022ad340..9207d15d7f 100644 --- a/tests/unit/test_materialization_core.py +++ b/tests/unit/test_materialization_core.py @@ -2394,7 +2394,7 @@ def test_roi_materialization_spec_reports_projected_source_identities() -> None: assert spec.emits_variable_component_planes(payload) is True assert [ identity.component_metadata - for identity in spec.emitted_source_identities(payload) + for identity in spec.stream_source_identities(payload) ] == [ { "well": "A01", From 91a374edd20c5c0e4f9cc1bc0baa30dbf0d6201a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 22:41:29 -0400 Subject: [PATCH 105/125] test: qualify original point writer domain and anchor across source selections --- ...live-fractional-z-point-projection-152.rst | 34 ++++ .../test_live_fractional_z_point_source.py | 168 ++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 tests/unit/test_live_fractional_z_point_source.py diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 574085c806..6a1ecf06c9 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -229,3 +229,37 @@ The preceding08 invocation rejected an invalid git revision WORKTREE before reading source; it is not an accepted census. This is an implemented source checkpoint awaiting bounded original-writer/projection regression verification; it is not installed live settlement or native reopening acceptance. + +Bounded original-writer/projection controls +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``tests/unit/test_live_fractional_z_point_source.py`` exercises seven controls +through a real declared PURE_3D tiny producer, the original point/CSV writers, +disk ZIP save/load, artifact source query, original viewer-backend kwargs and +unchanged Points geometric guard. Six positive cases cover singletonZ0, +four-planeZ0..3 andZ10..13, with direct and selected table payloads. A combined +CSV+point declaration retains one point archive while representing every source +plane. Exact first path/Z anchor and full unmodified archived provenance pass; +replacement preserves the nominal point output. Every positive also rejects +an out-of-domain coordinate through the original guard. Wrong payload type is +rejected by the original point declaration, used by writer and query alike. + +Control11 passed7 in11.16s, terminal0, maxRSS446292KiB, swaps0, scope512MiB, +noSwap/oneCPU/timeout60s. No native, viewer, catalog, Qt application or connection +was started. It uses an exported3a11b source snapshot with paired dependencies, +not a new installed target or environment. The reused native tabular binary is +from retained installed02; its original C++ source compares identical. The +snapshot avoids activating the five foreign dirty external checkouts and is +NOT an installed wheel/native acceptance claim. + +Original failed controls remain:08 selected the live checkout via working +directory and rejected stale-loaded externals;09 lacked pytest's configured +test-plugin import root;10 used a wrong ``path`` keyword in the test readback. +The corrected11 uses the original ``file_path`` contract. No production guard +or input assertion was weakened to pass these harness failures. + +Fresh after-inventory10 covers704 OpenHCS and95 paired PolyStore/ZMQ modules, +667 named owner/consumer sites, zero parse omissions (before09:661 sites). +The old emitted_source_identities declaration and production/test consumers +are gone; immutable original defect reproducer and historical receipts retain +that name as evidence. This named AST census is not a full NRA detector/proof. diff --git a/tests/unit/test_live_fractional_z_point_source.py b/tests/unit/test_live_fractional_z_point_source.py new file mode 100644 index 0000000000..64777323e9 --- /dev/null +++ b/tests/unit/test_live_fractional_z_point_source.py @@ -0,0 +1,168 @@ +"""Original point writer -> stream metadata -> unchanged projection controls. + +No compilation, viewer instance, native process or endpoint connection. +""" +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import load_rois_from_zip +from polystore.roi_converters import NapariROIConverter +from polystore.streaming.identity import StreamProducerIdentity +from polystore.streaming.viewer_transport import ViewerStreamProducer, ViewerStreamSourceIdentity +from polystore.streaming_constants import StreamingDataType +from zmqruntime.config import TransportMode +from zmqruntime.viewer_protocol import ViewerTransportEndpoint + +from openhcs.constants.constants import VariableComponents +from openhcs.core.config import NapariStreamingConfig +from openhcs.core.memory import numpy as numpy_func +from openhcs.core.measurement_row_materialization import MeasurementSparseColumnarRows +from openhcs.core.roi_point_metadata import ROIFractionalZ +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_measurements import ( + MeasurementScope, MeasurementSubject, MeasurementTable, ObjectCoreMeasurementFeature, +) +from openhcs.core.runtime_tabular_values import FieldSpec +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.steps.function_artifact_materialization import ArtifactStreamSourceMetadataAuthority +from openhcs.core.steps.stream_component_semantics import StreamComponentMessageExtraAuthority +from openhcs.core.streaming_config_declarations import ViewerType +from openhcs.core.streaming_config_factory import StreamingViewerRuntimeConfig, StreamingViewerSurface +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser +from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.processing.materialization import ( + CsvOptions, MaterializationSpec, PointROIOptions, materialization_outputs, materialize, +) +from openhcs.processing.materialization.core import PointROIOutput, ViewerStreamBackendCallKwargs +from openhcs.runtime.napari_streaming_handlers import NapariStreamLayerAddress, NapariStreamLayerItem +from openhcs.runtime.napari_viewer_server import NapariPointsLayerDisplayHandler +from openhcs.runtime.viewer_component_system import ViewerComponentValueDomainPayload + + +class SyntheticMicroscope: + """Original filename parser only; no metadata or runtime service needed.""" + + parser = SourceSchemaFilenameParser() + + +def point_fixture(root: Path, planes: int, origin: int): + components = tuple( + dict(well='A01', site=1, channel=1, z_index=origin + index, timepoint=1) + for index in range(planes) + ) + paths = tuple(str(root / f'A01_s001_w1_z{origin + index:03d}_t001.tif') for index in range(planes)) + volume = np.zeros((planes, 5, 7), dtype=np.uint16) + volume[:, 1:3, 2:4] = 1 + for path, plane in zip(paths, volume): + tifffile.imwrite(path, plane) + provenance = SourceImageProvenancePlanes.from_components(paths=paths, component_metadata=components) + + @numpy_func(contract=ProcessingContract.PURE_3D) + def synthetic_centres(image: np.ndarray) -> MeasurementTable: + z, y, x = np.argwhere(image > 0).mean(axis=0) + return MeasurementTable( + name='synthetic_centres', + rows=MeasurementSparseColumnarRows.from_rows( + (dict(object_label=7, center_z=float(z), center_y=float(y), center_x=float(x)),), + fields=(FieldSpec('object_label', int), FieldSpec('center_z', float), + FieldSpec('center_y', float), FieldSpec('center_x', float)), + ), + source_image_provenance_planes=provenance, + subject=MeasurementSubject(MeasurementScope.OBJECT, 'synthetic', 'object_label'), + ) + + return synthetic_centres(volume), components, paths + + +@pytest.mark.parametrize('planes,origin', ((1, 0), (4, 0), (4, 10))) +@pytest.mark.parametrize('selected', (False, True)) +def test_original_point_writer_live_domain_anchor_and_reopen(tmp_path, planes, origin, selected): + table, components, paths = point_fixture(tmp_path, planes, origin) + feature = ObjectCoreMeasurementFeature + options = PointROIOptions( + z_feature=feature.CENTER_Z, y_feature=feature.CENTER_Y, x_feature=feature.CENTER_X, + source='centres' if selected else None, + ) + payload = {'centres': table} if selected else table + spec = MaterializationSpec(CsvOptions(source=options.source), options) + fm = FileManager({'disk': DiskStorageBackend()}) + outputs = materialization_outputs( + spec, payload, str(tmp_path / 'centres'), fm, + variable_components=(VariableComponents.Z_INDEX,), + ) + point_output, = (output for output in outputs if isinstance(output, PointROIOutput)) + anchor = point_output.source_identity + assert anchor.path == paths[0] + assert dict(anchor.component_metadata) == components[0] + assert point_output.metadata.source_image_provenance_planes.count == planes + assert replace(point_output).source_identity == anchor + assert not spec.emits_variable_component_planes(payload) + identities = spec.stream_source_identities(payload) + assert tuple(identity.path for identity in identities) == paths + assert tuple(dict(identity.component_metadata) for identity in identities) == components + + archive = Path(materialize( + MaterializationSpec(options), data=payload, path=str(tmp_path / 'saved'), + filemanager=fm, backends=['disk'], backend_kwargs={}, + )) + rois = load_rois_from_zip(archive) + saved_metadata = ROIArchiveSourceMetadata.decode(rois) + saved_domain = ROIFractionalZ.source_component_domain(rois, saved_metadata) + assert tuple(dict(item) for item in saved_domain) == components + assert saved_metadata.source_image_provenance_planes == point_output.metadata.source_image_provenance_planes + if planes > 1: + assert 'z_index' not in (saved_metadata.source_component_metadata or {}) + + surface = StreamingViewerSurface( + runtime_config=StreamingViewerRuntimeConfig( + transport_endpoint=ViewerTransportEndpoint(host='127.0.0.1', port=6199, transport_mode=TransportMode.TCP), + persistent=True, viewer_type=ViewerType.NAPARI, + ), + display_config=NapariStreamingConfig(enabled=True), + source=ViewerStreamSourceIdentity(microscope_handler=SyntheticMicroscope(), plate_path=None), + ) + items = ArtifactStreamSourceMetadataAuthority.metadata_items( + materialization_spec=spec, data=payload, fallback_source_identity=None, + ) + authority = StreamComponentMessageExtraAuthority.from_viewer_surface(surface, source_metadata_items=items) + assert 'z_index' in authority.layout.component_order + producer = StreamProducerIdentity.pipeline_output( + output_kind='artifact', output_key='synthetic_centres', projection_key='synthetic_centres', + step_name='Own engineering only', pipeline_position=0, artifact_kind='roi', + ) + kwargs = ViewerStreamBackendCallKwargs(authority.viewer_backend_kwargs( + producer=ViewerStreamProducer.from_identity(producer), + )).to_filemanager_kwargs(point_output) + request = kwargs['stream_request'] + streamed = request.source.metadata.component_metadata_for_item(index=0, file_path=point_output.path) + assert int(streamed['z_index']) == origin + domain = ViewerComponentValueDomainPayload.from_wire_mapping( + authority.payload().to_wire_mapping()['component_value_domain'], context='own point source fixture', + ) + item = NapariStreamLayerItem( + data=NapariROIConverter.rois_to_shapes(rois), producer=producer, + address=NapariStreamLayerAddress(components=anchor.component_metadata, path=str(archive), stream_layer_data_type=StreamingDataType.POINTS), + image_metadata=saved_metadata, plane_component_domain=domain, + ) + handler = NapariPointsLayerDisplayHandler() + occupied = handler.geometric_component_values([item], authority.component_axis_semantics) + assert occupied == {'z_index': list(range(origin, origin + (3 if planes == 4 else 1)))} + with pytest.raises(ValueError, match='exceeds the declared Z domain'): + handler.geometric_component_values( + [replace(item, data=NapariROIConverter.rois_to_shapes([ + ROIFractionalZ(float(planes)).bind(rois[0]), + ]))], authority.component_axis_semantics, + ) + + +def test_point_source_admission_reuses_original_typed_payload_contract(): + feature = ObjectCoreMeasurementFeature + options = PointROIOptions(z_feature=feature.CENTER_Z, y_feature=feature.CENTER_Y, x_feature=feature.CENTER_X) + with pytest.raises(TypeError, match='requires a MeasurementTable'): + options.stream_source_identities(np.zeros((1, 5, 7))) From 1e6662632ab8ce84f121a6cb1f79e960df3bf906 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 22:43:04 -0400 Subject: [PATCH 106/125] docs: retain 32 point-source checks and current Root receiving boundary --- ...live-fractional-z-point-projection-152.rst | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 6a1ecf06c9..d677f4699c 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -263,3 +263,36 @@ Fresh after-inventory10 covers704 OpenHCS and95 paired PolyStore/ZMQ modules, The old emitted_source_identities declaration and production/test consumers are gone; immutable original defect reproducer and historical receipts retain that name as evidence. This named AST census is not a full NRA detector/proof. + +Regression and current integration disposition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Regression12 passed25 selected existing controls in9.37s, terminal0, +maxRSS452120KiB, swaps0, same512MiB/noSwap/oneCPU/60s scope. These cover original +point ZIP reopening and rejection, generic source-identity image files, file +bundles, TIFF stream/projection, singleton ROI metadata and filename identity. +47 unrelated cases were deselected; two pre-existing pytest async-configuration +warnings remain. This plus7 new controls is32 passes, not a full test suite or +installed native settlement acceptance. All owned scopes are terminal/inactive. + +After the fresh H002 stop was reported, determining remote heads were checked: +main9531d594,494 at91a374edd and Root394 still396379b. No new production change +or install is needed merely because the unchanged scientific498 fleet exhibits +the known failure. No scientist source, image, table or endpoint was inspected +or modified and no author was contacted. + +One targeted receiving diff of Root394 confirms that its MaterializationBatch +retains original typed Output objects through the same replace-based source +fallback/variable-component hooks and successful-save records. The narrow point +options, writer, source-identity property and artifact source-query seams remain +as previously released; Root's additional publication machinery does not yet +repair them. No generic provenance/compiler/load/stack patch is required here. +This receiving review is source evidence only, not execution against Root394. + +Remaining owner boundaries: Planck owns494 source closure/review and future +engineering point-producer acceptance. Parent owns integration/release of an +eligible ordinary target and a resource-admitted engineering endpoint, plus152's +continuous live/reopened native journey. Root retains134 broader persisted +publication. Scientific498 remains immutable. A later engineering journey must +exercise actual live point settlement and persisted reopening separately; these +are not inferred from source controls or an author's manual reopening. From fdad1567f15948527ab4d08e2868579219763431 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 22:52:24 -0400 Subject: [PATCH 107/125] docs: record current 3D GC attribution and producer ownership frontier Refs #384, #496. Keep scientific acceptance separate from diagnostic costs and document the existing producer semantics before the next generic runtime candidate. --- .../runtime_plumbing_frontier_20261003.rst | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/plans/runtime_plumbing_frontier_20261003.rst b/docs/plans/runtime_plumbing_frontier_20261003.rst index e34009a5cb..9335fe603f 100644 --- a/docs/plans/runtime_plumbing_frontier_20261003.rst +++ b/docs/plans/runtime_plumbing_frontier_20261003.rst @@ -132,3 +132,79 @@ PR #394 remains draft. Whole-branch original R0 and the #479 per-file R1 failures remain explicit and unwaived; installed consumer acceptance, full-catalog/scaling measurements and fresh figures remain unfinished. The performance goal remains active. + +Deeper current 3D attribution and producer ownership +--------------------------------------------------- + +Two further diagnostics use clean source ``396379b06769a3f5afe92413b7d9d6a21dd3f66f``. +Its production, test and benchmark driver bytes are unchanged from the preceding +``093c2f42`` source. Both preserve mandatory READY warmup, inline 1w_1t execution, +OUTCOMES export, the normal memory observer and both disabled profilers. Neither +is a new ordinary performance comparison. + +The 36-hook run separates metadata normalization and provenance operations from +their actual nested callers. It observes 29,701 normalization calls outside raw +callable boundaries, totaling 0.317682s exclusive. Ordinary normalization creates +a fresh scalar identity and reuses the cached plane collection; it does not copy +the complete 60-plane lineage. Whole-plane merges and derived naming do rebuild +plane/contributor snapshots. Those operations retain live-mutation and fresh +snapshot obligations. Their observed costs do not admit a standalone fix large +enough to close the 1.338s qualified execution gap. + +Several large isolated stalls prompted a separate 25-hook run with passive GC +callbacks. It does not disable GC, freeze objects, change thresholds, force a +collection or replace existing callbacks. Startup has no active pipeline root. +Its execution root is 8.462928s: 3.803686s within the 211 declared callable +boundaries and 4.659242s outside them. The public diagnostic clocks are execution +8.490154s, compilation 1.824986s and total 11.210223s. GC consumes 0.285505s, +including one 0.196025s generation-two collection during saving. Collection +durations overlap the owner clocks; adding them to plumbing would double count. +GC alone cannot explain or close the remaining gap, so a GC-policy optimization +is rejected. Different diagnostic clocks on identical production bytes are not +evidence of a production gain or regression. + +Both diagnostics match all six CSVs and 120 TIFFs byte for byte against the same +native-qualified baseline, with complete 128-file inventories and physical +input joins. No fresh native repetition is claimed. The 36-hook scientific +output is retained in a lossless, individually hash-verified archive at +``/var/tmp/openhcs-current-3d-owner-ledger-v2-20261003/scientific-output.tar.gz``; +its custody receipt records every original path, mode, size and hash. Later +readers must use or restore that archive rather than assume those extracted +files remain present. Previous baselines and frozen failures are unchanged. + +Source and consumer audits reject two tempting shortcuts. Artifact-only groups +already return ``NoMainFlowOutput`` and skip unstacking and saving. Canonical CP +publication already reuses the stored runtime artifact payload. Downstream CP +inputs deliberately distinguish stored secondary artifacts from relation-owned +current main flow. The latter has an independent mutable whole buffer and final +filename/source-component context. Repeated cache hits share that buffer. A +direct stored-artifact substitution would change these relationships and, in a +saved determining case, 62 metadata facts. The final named output bundle can +also retain earlier sibling outputs, so the last canonical return roster cannot +stand in for the complete main-flow cohort. + +The remaining structural route belongs to existing ``PatternGroupOutputData``, +``AlignedImageStack``, manifest and runtime-input owners: derive physical plane +views and final context from the exact correlated producer occurrence, preserving +the independent main-flow buffer, rather than repeatedly project and recompose +the whole cohort. The seven marked load/request/record/unstack/save/publication/ +reconciliation spans total 3.432617s in the GC diagnostic. This is an upper +envelope containing mandatory work, not a claimed removable cost. Before a +candidate, representative replay must retain the stored occurrence, actual +merged bundle, projected leaves, final records and cache, filename/path prestates, +and next consumer edge in one graph. No fake processing context, new cache, +canonical-roster shortcut or zero-copy assumption is admitted. + +Additional retained receipts: + +* ``/var/tmp/openhcs-current-3d-owner-ledger-v2-scientific-byte-gate-20261003.json``: + SHA-256 ``df4f6548f4459c6a55ea5dc182ac52a5472eaacace572ae60740a9409377ca89``. +* ``/var/tmp/openhcs-current-3d-gc-ledger-v1-scientific-byte-gate-20261003.json``: + SHA-256 ``d0b8d6af1b8a1777b03624cf6435d67b04746e9477a5cefbc7a962feaa2d667d``. +* ``/var/tmp/openhcs-current3d-canonical-mainflow-CP-owner-review-v1-20261003.json``: + SHA-256 ``9c264eaa2a735388f5694a305dcbf8ff61258ed959adb4819ffacfcb39235bbf``. +* ``/var/tmp/openhcs-current3d-canonical-mainflow-macro-owner-audit-v4-20261003.json``: + SHA-256 ``c589df6f3c28d6d3a3dc668adc696089bc4aeb68aa2d762a0128592e8d67b891``. + +This checkpoint changes documentation only. No optimization, merge, original +R0/R1 waiver, installed acceptance or full-catalog/scaling completion is claimed. From 75cffb0aa093bc018bbe2e71082019877cb4063a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 22:55:01 -0400 Subject: [PATCH 108/125] docs: record custody of the completed GC diagnostic outputs Refs #384. All scientific bytes remain in individually verified lossless archives; no source or benchmark behavior changed. --- docs/plans/runtime_plumbing_frontier_20261003.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/plans/runtime_plumbing_frontier_20261003.rst b/docs/plans/runtime_plumbing_frontier_20261003.rst index 9335fe603f..7d5eb76dbe 100644 --- a/docs/plans/runtime_plumbing_frontier_20261003.rst +++ b/docs/plans/runtime_plumbing_frontier_20261003.rst @@ -170,7 +170,9 @@ output is retained in a lossless, individually hash-verified archive at ``/var/tmp/openhcs-current-3d-owner-ledger-v2-20261003/scientific-output.tar.gz``; its custody receipt records every original path, mode, size and hash. Later readers must use or restore that archive rather than assume those extracted -files remain present. Previous baselines and frozen failures are unchanged. +files remain present. The 25-hook output uses the same verified custody protocol +at ``/var/tmp/openhcs-current-3d-gc-ledger-v1-20261003/scientific-output.tar.gz``. +Previous baselines and frozen failures are unchanged. Source and consumer audits reject two tempting shortcuts. Artifact-only groups already return ``NoMainFlowOutput`` and skip unstacking and saving. Canonical CP From dc01ee53a81f280822c62a762b9325a931a39f67 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 23:04:13 -0400 Subject: [PATCH 109/125] docs: qualify ordinary installed fractional-Z point checkpoint --- ...live-fractional-z-point-projection-152.rst | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index d677f4699c..d55937ca9e 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -296,3 +296,30 @@ continuous live/reopened native journey. Root retains134 broader persisted publication. Scientific498 remains immutable. A later engineering journey must exercise actual live point settlement and persisted reopening separately; these are not inferred from source controls or an author's manual reopening. + +Ordinary installed packaging checkpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reviewed main e9e79ac41 was normally merged at39244bab1d47d5af3c87ff96f70014ff091c67ee; +the three reviewed point repair files and original guard remain unchanged. +The ordinary wheel and new private target are retained under +``/home/ts/wt/openhcs-issue-batch-20260929/engineering494`` (wheels01/target01). +Wheel SHA256524cb5262f7fa2f396c5018e4cce22bf0e13847d58ea212200cf43ea1b78bdf9. +The original setup hooks, existing paired interpreter and no-dependency pip +route were used, without a new environment, checkout or download. + +The original wheel verifier and current declaration-owned knowledge projection +confirm815 tracked OpenHCS/benchmark source/wheel/target byte matches,90 declared +knowledge/skill assets, all13 skill files and both native C++/ABI3 extensions. +Actual installed-owner origin assertions preceded seven passing original tiny +writer/projection controls; two pre-existing pytest configuration warnings +remain. These exercise real disk ZIP save/readback and unchanged geometric +guards, not an MCP/native viewer. Build/install/installed-control cgroup peaks +were283799552/106414080/378204160B respectively, under512MiB/noSwap/CPU1/60s; +all terminal0, swap/OOM0, scopes inactive. Five foreign gitlinks remain dirty +and untouched. Full hashes, commands, ownership and release boundaries are in +``engineering494/QUALIFICATION01.rst`` and original logs. + +Source and installed writer/projection behavior are ready. Actual public MCP +live settlement and persisted reopening/alignment remain unverified until +parent releases one exact engineering slot; no runtime/viewer was launched. From 436428228e053986bb07c2e8607b8ab3e04f6761 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Fri, 2 Oct 2026 23:29:29 -0400 Subject: [PATCH 110/125] docs: retain prepared bounded public point acceptance journey --- ...live-fractional-z-point-projection-152.rst | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index d55937ca9e..65cce4fab5 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -323,3 +323,23 @@ and untouched. Full hashes, commands, ownership and release boundaries are in Source and installed writer/projection behavior are ready. Actual public MCP live settlement and persisted reopening/alignment remain unverified until parent releases one exact engineering slot; no runtime/viewer was launched. + +Public journey preparation checkpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Parent named the future existing H00294 helpers (VNC5998), native6012/7012 +and viewer6013/7013; custody remains scientific and NOT released at preparation. +Complete ``pipeline-live01.py``, reviewed ``point_volume_fixture_494.py``, +global config and original journalled ``launch-live01.sh`` are now saved in +engineering494. ``LIVE01-PREPARATION.rst`` records the ordered producer, +automatic-settlement, same-viewer persisted reopening, calibration/feature +linkage and genuine XY/XZ/YZ acceptance journey. Only the original four own +5x7 uint16 source planes are admitted; source-domain/metadata construction +remains with the original declarations and runtime. No SCI source is consulted. + +Stdlib AST/JSON and shell syntax checks pass (``preparation01.log``); this is +NOT registration, public validation/compile or native acceptance. No project +imports/client/native/viewer start occurs. Runtime remains one aggregate4GiB, +noSwap/CPU1, ordinary10s observations and original handle custody. Packaged +target01 stays unchanged and requires no rebuild for this documentation-only +checkpoint. Actual slot/resource admission and every live contract are pending. From 87c841f16798a14e29ef2e96736e43122966f263 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:02:03 -0400 Subject: [PATCH 111/125] Derive fractional-Z point domain checks from component owners --- ...live-fractional-z-point-projection-152.rst | 57 ++++++++++++++++ openhcs/core/roi_point_metadata.py | 21 +++++- .../test_live_fractional_z_point_source.py | 67 ++++++++++++++++++- 3 files changed, 139 insertions(+), 6 deletions(-) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 65cce4fab5..37954ecb5d 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -343,3 +343,60 @@ imports/client/native/viewer start occurs. Runtime remains one aggregate4GiB, noSwap/CPU1, ordinary10s observations and original handle custody. Packaged target01 stays unchanged and requires no rebuild for this documentation-only checkpoint. Actual slot/resource admission and every live contract are pending. +Actual public failure03 and source-owner repair04 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Parent released the original programme on recovered94. Original client +PTY39222/InvocationID1a31b8490e614741865cca14e6775244/start1790998807 ran +unchanged target01. The read-only original input could not admit workspace +initialization; four326B TIFFs were staged byte-identically under admitted +output/input-staged01, preserving original plate_path through the public +execution_plate_path contract. The original rejection and session-1 remain; +only distinct session-2 compiled and executed. No source or registration replay. + +Compile871111d1-764e-434f-a8f9-e2c22329839d COMPLETE. Single public execution +113462a1-229b-478a-b4a5-0e1cd1a53cb7 FAILED in original point writer/core3268, +ROIFractionalZ.source_component_domain92: ``3D point ROI source planes vary +outside Z.`` This occurred before returning PointROIOutput/point settlement. +One main image route streamed; no point archive or runtime-values export was +completed. Actual raw PNG opened personally. Point triples/calibration/row +alignment/reopen are MISSING, not a native acceptance. All original journals, +full trace, PNG and typed closes are frozen in engineering494/LIVE03-TERMINAL.rst +and LIVE03-FROZEN.sha256. Native2005980/create1790998932.22 and viewer2039581/ +create1790999389.18 both public ACK/process_exited/endpoint_terminated/succeeded +true; four ports and all owned PIDs gone, original scope inactive. Client exit1 +is retained (earlier strict argument rejections), separate from failed science. +Peak2403676160B/noSwap under4GiB. Helpers and three SCI remain untouched. + +Actual public compiled source facts are retained in +engineering494/PUBLIC-PLANE-FACTS03.json. The original guard's only non-Z raw-map +disagreements are OriginalSourceMetadata containing literal Z000..003 and +SourceFilterPaths containing each distinct original TIFF path. Canonical +well/site/channel/time and source calibration agree; Z is ordered0..3. Failed +runtime table metadata was not exported, so this is compiled-source evidence, +not a fabricated recovered runtime table. Original source flow passes complete +plane provenance through the measurement projection to the original writer. + +This is IDEN-7 (a check wider than the coordinate question) and BOUND-2 (bypassing +the existing component owner). Fix ONLY ROIFractionalZ.source_component_domain: +derive coordinates through source_component_metadata_items and its original +SourceComponentProjectionStrategy/SourceMetadataRoleView, compare nominal +non-Z membership/values, validate uniform SourceVoxelSpacing through its owner, +and return the FULL original domain unchanged. Delete raw all-key equality; +do not discard provenance, duplicate a component roster, fabricate canonical +metadata, modify compiler/source construction/stack, or weaken Z bounds. +Original writer and persisted-reopen service already consume this one guard. +Necessary exact seam was posted to Root394, comment5965305655. Parent explicitly +released source-only494 repair; no new runtime/wheel release is inferred. + +Original refactor-audit AST traversal was reused file-at-a-time, including +registered coordinate/role/calibration families and every guard consumer: +engineering494/AST-FAMILY04.json,704 OpenHCS and95 actual paired PolyStore/ZMQ +modules, no parse omissions. Named AST coverage is not a global NRA proof or +dynamic equivalence proof. Current main87d9a99a integrated normally; determining +#509 changes only knowledge-service/test, no point production seam. All five +foreign dirty gitlinks and immutable installed target01 are untouched. + +Coherent source correction and expanded original writer/reopen/anchor rejection +controls are prepared. Bounded source validation follows this checkpoint; +neither its result nor a future installed/native acceptance is claimed yet. diff --git a/openhcs/core/roi_point_metadata.py b/openhcs/core/roi_point_metadata.py index f597dd2bec..90ef89c57e 100644 --- a/openhcs/core/roi_point_metadata.py +++ b/openhcs/core/roi_point_metadata.py @@ -11,6 +11,8 @@ from openhcs.constants.constants import AllComponents from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.source_image_provenance import SourceComponentMetadata +from openhcs.core.source_matching import source_component_metadata_items +from openhcs.core.source_metadata import SourceVoxelSpacing @dataclass(frozen=True, slots=True) @@ -83,11 +85,24 @@ def source_component_domain( raise ValueError( "3D point ROI Z coordinate lies outside its source planes." ) - common_keys = set(domain[0]) - {z_field} - if any(set(plane) - {z_field} != common_keys for plane in domain): + components = tuple( + dict(source_component_metadata_items(plane)) for plane in domain + ) + common_keys = set(components[0]) - {AllComponents.Z_INDEX} + if any( + set(plane) - {AllComponents.Z_INDEX} != common_keys + for plane in components + ): raise ValueError("3D point ROI source planes have inconsistent components.") if any( - any(plane[key] != domain[0][key] for key in common_keys) for plane in domain + any(plane[key] != components[0][key] for key in common_keys) + for plane in components ): raise ValueError("3D point ROI source planes vary outside Z.") + spacing = SourceVoxelSpacing.from_source_metadata(domain[0]) + if any( + SourceVoxelSpacing.from_source_metadata(plane) != spacing + for plane in domain + ): + raise ValueError("3D point ROI source planes have inconsistent calibration.") return tuple(domain) diff --git a/tests/unit/test_live_fractional_z_point_source.py b/tests/unit/test_live_fractional_z_point_source.py index 64777323e9..51b8db4723 100644 --- a/tests/unit/test_live_fractional_z_point_source.py +++ b/tests/unit/test_live_fractional_z_point_source.py @@ -30,6 +30,9 @@ ) from openhcs.core.runtime_tabular_values import FieldSpec from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import ( + OriginalSourceMetadata, SourceFilterPathMetadata, SourceVoxelSpacing, +) from openhcs.core.steps.function_artifact_materialization import ArtifactStreamSourceMetadataAuthority from openhcs.core.steps.stream_component_semantics import StreamComponentMessageExtraAuthority from openhcs.core.streaming_config_declarations import ViewerType @@ -51,12 +54,22 @@ class SyntheticMicroscope: parser = SourceSchemaFilenameParser() -def point_fixture(root: Path, planes: int, origin: int): +def point_fixture(root: Path, planes: int, origin: int, *, source_records=False): components = tuple( dict(well='A01', site=1, channel=1, z_index=origin + index, timepoint=1) for index in range(planes) ) paths = tuple(str(root / f'A01_s001_w1_z{origin + index:03d}_t001.tif') for index in range(planes)) + if source_records: + for component, path in zip(components, paths, strict=True): + OriginalSourceMetadata.from_mapping({ + 'well': 'A01', 'site': '001', 'channel': '1', + 'timepoint': '001', 'z_index': f"{component['z_index']:03d}", + }).merge_into(component, path=path) + SourceFilterPathMetadata.from_paths((Path(path).name, path)).merge_into( + component, path=path, + ) + SourceVoxelSpacing((2.0, 0.65, 0.65)).merge_into(component, path=path) volume = np.zeros((planes, 5, 7), dtype=np.uint16) volume[:, 1:3, 2:4] = 1 for path, plane in zip(paths, volume): @@ -82,8 +95,9 @@ def synthetic_centres(image: np.ndarray) -> MeasurementTable: @pytest.mark.parametrize('planes,origin', ((1, 0), (4, 0), (4, 10))) @pytest.mark.parametrize('selected', (False, True)) -def test_original_point_writer_live_domain_anchor_and_reopen(tmp_path, planes, origin, selected): - table, components, paths = point_fixture(tmp_path, planes, origin) +@pytest.mark.parametrize('source_records', (False, True)) +def test_original_point_writer_live_domain_anchor_and_reopen(tmp_path, planes, origin, selected, source_records): + table, components, paths = point_fixture(tmp_path, planes, origin, source_records=source_records) feature = ObjectCoreMeasurementFeature options = PointROIOptions( z_feature=feature.CENTER_Z, y_feature=feature.CENTER_Y, x_feature=feature.CENTER_X, @@ -166,3 +180,50 @@ def test_point_source_admission_reuses_original_typed_payload_contract(): options = PointROIOptions(z_feature=feature.CENTER_Z, y_feature=feature.CENTER_Y, x_feature=feature.CENTER_X) with pytest.raises(TypeError, match='requires a MeasurementTable'): options.stream_source_identities(np.zeros((1, 5, 7))) + + +@pytest.mark.parametrize('field,value,error', ( + ('well', 'B01', 'vary outside Z'), + ('site', 2, 'vary outside Z'), + ('channel', 2, 'vary outside Z'), + ('timepoint', 2, 'vary outside Z'), + ('well', None, 'inconsistent components'), + ('site', None, 'inconsistent components'), + ('channel', None, 'inconsistent components'), + ('timepoint', None, 'inconsistent components'), + ('z_index', 0, 'consecutive and ordered'), + ('z_index', 3, 'consecutive and ordered'), + ('z_index', None, 'require z_index'), + ('z_index', True, 'must be integers'), + ('z_index', 1.5, 'must be integers'), + ('z_index', '1.5', 'must be integers'), + ('OpenHCSSourceVoxelSpacingZYX', '2,0.7,0.7', 'inconsistent calibration'), + ('OpenHCSSourceVoxelSpacingUnit', 'relative', 'inconsistent calibration'), +)) +def test_point_domain_preserves_real_coordinate_and_calibration_rejections(tmp_path, field, value, error): + table, components, paths = point_fixture(tmp_path, 4, 0, source_records=True) + invalid = tuple(dict(plane) for plane in components) + if value is None: + del invalid[1][field] + else: + invalid[1][field] = value + metadata = ImagePayloadMetadata( + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=paths, component_metadata=invalid, + ), + ) + from polystore.roi import ROI, PointShape + rois = [ROIFractionalZ(1.5).bind(ROI(shapes=[PointShape(y=1.5, x=2.5)]))] + with pytest.raises(ValueError, match=error): + ROIFractionalZ.source_component_domain(rois, metadata) + + +@pytest.mark.parametrize('coordinate', (-0.5, 3.5)) +def test_point_domain_preserves_fractional_z_bounds_with_literal_provenance(tmp_path, coordinate): + table, components, paths = point_fixture(tmp_path, 4, 0, source_records=True) + from polystore.roi import ROI, PointShape + rois = [ROIFractionalZ(coordinate).bind(ROI(shapes=[PointShape(y=1.5, x=2.5)]))] + with pytest.raises(ValueError, match='outside its source planes'): + ROIFractionalZ.source_component_domain( + rois, ImagePayloadMetadata(source_provenance=table.source_provenance), + ) From 4201b0f91c50f7a277012a434731fee73f2f583c Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:08:26 -0400 Subject: [PATCH 112/125] Record failed public journey and exact nested identity dependency --- ...live-fractional-z-point-projection-152.rst | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 37954ecb5d..1c2bb6b2cf 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -397,6 +397,22 @@ dynamic equivalence proof. Current main87d9a99a integrated normally; determining #509 changes only knowledge-service/test, no point production seam. All five foreign dirty gitlinks and immutable installed target01 are untouched. -Coherent source correction and expanded original writer/reopen/anchor rejection -controls are prepared. Bounded source validation follows this checkpoint; -neither its result nor a future installed/native acceptance is claimed yet. +Published source87c841f16 independently reproduces the original rejection from +EXACT retained public plane records at unchanged target01, then accepts the +same full unmodified records at corrected source. Original reproduction +terminal0/32,124,928B/noSwap/OOM0. Source05 controls:25 PASS/6 FAIL in4.36s, +terminal1/319,324,160B/512MiB/noSwap/CPU1/60s/OOM0. Original test-plugin harness +failure04 retained; distinct05 adds only the actual test-package import root. + +Passing25 retain genuine non-Z/invalid/duplicate/disordered/fractional Z, +calibration/unit, original writer/ZIP/anchor/projection and payload guards. +Six enriched-source writer/ZIP checks preserve every original metadata value, +but full provenance identity differs after nested JSON keys reorder. No assertion +is relaxed or fixture artificially sorted. Root-owned +source_image_provenance._component_metadata_identity uses repr of nested raw +maps; existing SourceMetadataIdentityProjection already owns canonical nested +identity. Exact delegation hunk and six failures posted Root394 +comment5965364042, not patched across its protected source owner. Named remaining +dependency is that precise seam release/owner implementation. Full receipt: +engineering494/SOURCE04-QUALIFICATION.rst. No fresh wheel/runtime or installed +native acceptance is claimed; original03 stays FAILED/frozen. From 054bf7b643a76d78ef8ddcbf6d60d7db212fda33 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:21:48 -0400 Subject: [PATCH 113/125] Qualify point domain rejection subset without claiming full family pass --- .../live-fractional-z-point-projection-152.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 1c2bb6b2cf..fc5bb96722 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -416,3 +416,17 @@ comment5965364042, not patched across its protected source owner. Named remainin dependency is that precise seam release/owner implementation. Full receipt: engineering494/SOURCE04-QUALIFICATION.rst. No fresh wheel/runtime or installed native acceptance is claimed; original03 stays FAILED/frozen. + +Separate domain/rejection qualification06 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Distinct retained-source check selects only point_source_admission and +point_domain_preserves:19 PASS/12 DESELECTED in4.24s, terminal0. Actual guard +origin is source04/openhcs/core/roi_point_metadata.py. MemoryMax512MiB, +peak307187712B/noSwap/CPU1/60s/OOM0; scope terminal/inactive. This proves those +admission/domain rejections only, NOT full31PASS or installed native acceptance. +Original25PASS/6FAIL and all enriched writer/ZIP identity failures remain +unchanged. Root394 bb9c has no release for comment5965364042; shared source is +not patched. Original failed public journey, logs, source and scratch preserved. +Evidence: engineering494/domain-rejection-control06.py and matching log, +DOMAIN06-GUIDE06-QUALIFICATION.rst. No build or service launched. From 03825ca3132b89e4a65bf58b2b3404fc15f838bf Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:21:48 -0400 Subject: [PATCH 114/125] Clarify matched point-centre viewer QA in canonical skill guide --- .../use-openhcs/references/viewer-qa.md | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/packaging/codex/openhcs/skills/use-openhcs/references/viewer-qa.md b/packaging/codex/openhcs/skills/use-openhcs/references/viewer-qa.md index 823cdd33b3..a3cd7d588c 100644 --- a/packaging/codex/openhcs/skills/use-openhcs/references/viewer-qa.md +++ b/packaging/codex/openhcs/skills/use-openhcs/references/viewer-qa.md @@ -41,15 +41,6 @@ If remote-desktop compression is suspected, open a native MCP PNG capture at the same viewport before attributing absent paths to the transport. Distinguish hidden raw signal from paths actually missing in a saved processing stage. -For feature-bearing 3-D point results, check the persisted point coordinates -against the viewer's native point geometry, declared Z domain, and selected -feature row. Select a point by its data index through MCP and verify that the -same row and exact Z coordinate remain attached. A result-only Points layer -may have a native slice range determined solely by its points, so a displayed -integer Z label or `current_step` alone does not establish the point's Z or -its alignment with raw planes. Confirm that alignment in the combined view; -record any disagreement rather than treating a visible point as a QA pass. - ## Choose a distributed, multiscale sample Inspect the whole field raw-only, with results hidden, for illumination, tissue, @@ -131,6 +122,38 @@ changed during capture, re-establish state and recapture the matched set before comparison. Repeat at necessary field, context and object scales; no single view is an acceptance witness. +### Points and centres + +Point results need the same three-view comparison; selected raw-plus-points +captures or point counts alone are insufficient. Identify the intended final +Points layer and its persisted result/source identity, excluding earlier result +versions and unrelated Labels, Shapes or Points from the matched set. + +- **Raw only:** hide the point result and other result layers. +- **Point only:** hide **every image layer**, not just the selected raw image, + and show only the intended final Points result. Keep raw mounted but hidden + so its source/domain context remains available; do not unload it for this + comparison. +- **Combined:** restore the matching raw image and that same Points layer, + retaining the raw window and the set's native coordinates, camera and axes. + +For feature-bearing points, select by the final layer's `data_index` through +MCP and verify the attached feature row against the persisted point and native +geometry. For 3-D results record the exact, possibly fractional, Z coordinate +and declared source Z domain, not only the displayed integer slice label. +Visibility or selection changes can alter a point-only view's native slice +range. Read back camera, axes and coordinates after each transition; restore +the matched view if they changed. Keeping raw mounted is not proof that the +viewer preserved that state. Inspect genuine XY/XZ/YZ views when the source +and exposed contracts support them; record unavailable views without inventing +a projection or source domain. + +Open all three captures and judge raw support, placement and separation at +distributed positions and scales. Relate points to masks or structures only +when that association belongs to the intended measurement; there is no +universal requirement that every centroid lie inside a mask. Record geometry, +row or alignment disagreements instead of treating a visible point as a pass. + ## Diagnose one failure and preserve a regression control Log a clear positive, a plausible miss/ambiguity and an explicit From 4eaacf17e5de3dba59ec217f133fa6e1821c8641 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:23:35 -0400 Subject: [PATCH 115/125] docs: pin current runtime replay and compilation frontiers (refs #384, #496) --- .../runtime_plumbing_frontier_20261003.rst | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/plans/runtime_plumbing_frontier_20261003.rst b/docs/plans/runtime_plumbing_frontier_20261003.rst index 7d5eb76dbe..a41e346ec9 100644 --- a/docs/plans/runtime_plumbing_frontier_20261003.rst +++ b/docs/plans/runtime_plumbing_frontier_20261003.rst @@ -210,3 +210,79 @@ Additional retained receipts: This checkpoint changes documentation only. No optimization, merge, original R0/R1 waiver, installed acceptance or full-catalog/scaling completion is claimed. + +Current producer/consumer evidence and compilation frontier +---------------------------------------------------------- + +Fresh diagnostics at ``bb9c49c2ad70f1cee49fa0cde47592681edb2be3`` preserve +main ``87d9a99a9``, shared dependency/native/input pins, normal READY warmup, +inline 1w_1t execution, OUTCOMES completion and the normal memory observer. +Their observation clocks are not ordinary performance improvements. + +The bounded V7 capture retains four actual step-13 ImageMath producer and +step-14 Resize load states, totaling 856,419,294 bytes under the unchanged +384 MiB per-item and 1 GiB total limits. Its full six-CSV/120-TIFF parity, +complete scientific inventory, all 180 ordered physical source references, +and three native input-volume witnesses pass. Original step-13 replay matches +all 60 records, named/VFS/cache outputs and the complete 123-array identity, +sharing, dtype, stride and writeability graph. + +Original step-14 replay is RED before cache lookup: the actual producer +manifest lives in module ``_STEP_OUTPUT_MANIFESTS``, a WeakKeyDictionary outside +ProcessingContext transport. Reconstructing it from the separate producer +oracle would fabricate a receiver state and is rejected. Independently, +RuntimeSourceBindingContextCache deliberately drops four derived caches through +its reducer. The actual after-load graph proves the result is the retained +whole image for its exact 60-path cache key, but cold restored clocks cannot +represent warm pipeline work. A future replay must carry actual manifest/cache +owners in the same graph and rekey identity indexes only from real owner +references, or obtain live production confirmation. Original failed artifacts +remain preserved; schema completeness is not complete transaction state. + +A six-group ephemeral layout experiment validates the proposed existing-class +projection ancestor layout, constructor signatures, default factories, frozen +assignment, cooperative projection order, errors and fresh public provenance. +This proves layout feasibility only. Lazy read algorithms, custom subclass +behavior, combined transaction acceptance and sufficient elapsed savings +remain unproved. No new wrapper or cache authority is implemented. + +Illumination remains the weakest known total comparison. Its current V3 +diagnostic records compilation 1.125192s, execution job 0.377852s and total +1.844212s, with both authored NPY files, full inventory and actual physical +TIFF/pipeline/native witnesses matching the retained qualified reference. +The compilation partition places 0.302983s exclusive in dataclass analysis +and 0.285258s in nested documentation extraction: 0.588241s together. Four +client pipeline renders plus three config renders take 0.235231s. The joint +0.823472s envelope contains required work; it is not a saving estimate. +Ten effective-config getters include the schema/state subtree and cannot be +added to it. Completion alone is too small to close the remaining gap. + +The separate V4 diagnostic also passes full science and distinguishes actual +cache topology: compilation has 40 successful first-use misses and 338 hits +across 44 class identities; execution has 94 hits. There are no uncached empty +results. This rejects repeated failing analysis or fresh proxy churn as the +mechanism. Exact runtime class names were not retained because the diagnostic +read a type-owned attribute from the class dictionary; that metadata omission +is explicit. A separate declaration inspection finds 22 registered public/proxy +pairs and 95 differing fields, including Optional annotations and None defaults. +Canonical-class ParameterInfo substitution would change real reset and +reconstruction consumers, so it is rejected. The remaining candidate must +separate reusable declaration work from correctly timed value/default-factory +evaluation using the existing schema and preparation owners. + +Local source-qualified receipts: + +* ``/home/ts/.local/state/openhcs-maintenance/20261003/current-cohort-capture-v7-science-verification-v2.json`` + (SHA-256 ``99f2f830861eb1ecd1492c50ff6bbb2a66172674211e0a925f9f6a85359a1be0``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/current-cohort-v7-original-replay-v3.json``. +* ``/home/ts/.local/state/openhcs-maintenance/20261003/current-cohort-v7-load-owned-role-review.json`` + (SHA-256 ``bd08002bbb535911ba189b1b97bf8cbceb03c938b89c5ec8d0c061e4743735a7``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/projection_ancestor_feasibility_bb9c_v3.json`` + (SHA-256 ``b5ff353e33c70d4066de5315c0bef2046e1e8680056125404a7c393eb2e531ae``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/illumination-compiler-ledger-v3-science.json`` + (SHA-256 ``e042e25aa0fd9e087459c77613150f0db6447e91d0fc943acf2ea638312cc850``). +* ``/home/ts/.local/state/openhcs-maintenance/20261003/illumination-schema-cache-topology-v4-20261003.json`` + (SHA-256 ``134244ea39004ccfc86d20fa14d4fdc49e78aa72f4a92349092c6f420c1b89e3``). + +No production optimization, new native repetition, original R0/R1 waiver, +full-catalog/scaling completion or accepted ordinary speedup is claimed here. From 02e4967533a43061bbd0c7e31a96429be285752d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:28:12 -0400 Subject: [PATCH 116/125] Retain mapping-equivalence and real provenance difference controls --- .../test_source_provenance_nested_identity.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/unit/test_source_provenance_nested_identity.py diff --git a/tests/unit/test_source_provenance_nested_identity.py b/tests/unit/test_source_provenance_nested_identity.py new file mode 100644 index 0000000000..a4f234c4b9 --- /dev/null +++ b/tests/unit/test_source_provenance_nested_identity.py @@ -0,0 +1,54 @@ +"""Metadata mapping order is not identity; values and plane order are.""" +from collections.abc import Mapping + +from openhcs.core.source_image_provenance import SourceImageIdentity, SourceImageProvenancePlanes +from openhcs.core.source_metadata import OriginalSourceMetadata, SourceFilterPathMetadata + + +def source_records(): + metadata = {"well": "A01", "site": "1", "channel": "1", "z_index": "0"} + OriginalSourceMetadata.from_mapping({"well": "A01", "site": "001", "z_index": "000"}).merge_into( + metadata, path="/input/A01_z000.tif", + ) + SourceFilterPathMetadata.from_paths(("A01_z000.tif", "/input/A01_z000.tif")).merge_into( + metadata, path="/input/A01_z000.tif", + ) + return metadata + + +def test_source_identity_nested_mapping_reorder_is_equivalent(): + metadata = source_records() + reordered = { + key: dict(reversed(tuple(value.items()))) if isinstance(value, Mapping) else value + for key, value in reversed(tuple(metadata.items())) + } + before = SourceImageIdentity("/input/A01_z000.tif", metadata) + after = SourceImageIdentity("/input/A01_z000.tif", reordered) + assert dict(before.component_metadata) == metadata + assert dict(after.component_metadata) == reordered + assert before.identity == after.identity + planes = SourceImageProvenancePlanes.from_components + assert planes(paths=(before.path,), component_metadata=(metadata,)) == planes( + paths=(after.path,), component_metadata=(reordered,), + ) + + +def test_source_identity_changed_nested_value_is_distinct(): + original = source_records() + changed = source_records() + OriginalSourceMetadata.from_mapping({"well": "A01", "site": "002", "z_index": "000"}).merge_into( + changed, path="/input/A01_z000.tif", + ) + assert SourceImageIdentity("/input/A01_z000.tif", original).identity != SourceImageIdentity( + "/input/A01_z000.tif", changed, + ).identity + + +def test_source_identity_distinct_plane_order_is_not_equivalent(): + first = SourceImageIdentity("/input/A01_z000.tif", source_records()) + second = SourceImageIdentity("/input/A01_z001.tif", {"well": "A01", "z_index": "1"}) + planes = SourceImageProvenancePlanes.from_components + forward = planes(paths=(first.path, second.path), component_metadata=(first.component_metadata, second.component_metadata)) + reverse = planes(paths=(second.path, first.path), component_metadata=(second.component_metadata, first.component_metadata)) + assert forward.identity != reverse.identity + assert forward != reverse From 04ce7edb0973a5142859795fad5ba483a2099456 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:31:02 -0400 Subject: [PATCH 117/125] Record integrated Root identity failure and exact receiving-owner boundary --- ...live-fractional-z-point-projection-152.rst | 21 +++++++++++++++++++ .../test_source_provenance_nested_identity.py | 9 +++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index fc5bb96722..993b17d2b1 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -430,3 +430,24 @@ unchanged. Root394 bb9c has no release for comment5965364042; shared source is not patched. Original failed public journey, logs, source and scratch preserved. Evidence: engineering494/domain-rejection-control06.py and matching log, DOMAIN06-GUIDE06-QUALIFICATION.rst. No build or service launched. + +Receiving Root owner integration07 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The next determining owner check found Root394 advanced to4eaacf17e5de before +any helper/import edit. Normal merge337a31c9b receives its existing +SourceMetadataFields owner unchanged. No competing production hunk authored. +Its provenance_identity_items still explicitly preserves nested insertion-order +repr; adjacent identity_items owns nested key canonicalization. Exact remaining +owner method hunk posted394 comment5965533217 for Root implementation/release. + +Named-family AST update704 OpenHCS+95 paired modules, zero parse omissions. +Actual source07 batch:27PASS/7FAIL in4.85s, terminal1, peak298295296B/512MiB, +Swap0/CPU1/60s/OOM0. Original31 remains25PASS/6FAIL unchanged; three additional +identity controls give2PASS/1FAIL: changed nested values and plane order remain +distinct, equivalent reordered mappings fail. This is NOT repaired identity, +full31PASS, wheel/native settlement or biological acceptance. All original six +failures and public journals preserved. No assertion weakening/fixture sorting, +live target changes, new runtime, build or environment. Five foreign gitlinks +untouched. Evidence engineering494/ROOT07-QUALIFICATION.rst, +IDENTITY-FAMILY07.json, source-control07.py/.log and ROOT-IDENTITY-HUNK07.rst. diff --git a/tests/unit/test_source_provenance_nested_identity.py b/tests/unit/test_source_provenance_nested_identity.py index a4f234c4b9..12516f33a9 100644 --- a/tests/unit/test_source_provenance_nested_identity.py +++ b/tests/unit/test_source_provenance_nested_identity.py @@ -5,9 +5,9 @@ from openhcs.core.source_metadata import OriginalSourceMetadata, SourceFilterPathMetadata -def source_records(): +def source_records(*, literal_site="001"): metadata = {"well": "A01", "site": "1", "channel": "1", "z_index": "0"} - OriginalSourceMetadata.from_mapping({"well": "A01", "site": "001", "z_index": "000"}).merge_into( + OriginalSourceMetadata.from_mapping({"well": "A01", "site": literal_site, "z_index": "000"}).merge_into( metadata, path="/input/A01_z000.tif", ) SourceFilterPathMetadata.from_paths(("A01_z000.tif", "/input/A01_z000.tif")).merge_into( @@ -35,10 +35,7 @@ def test_source_identity_nested_mapping_reorder_is_equivalent(): def test_source_identity_changed_nested_value_is_distinct(): original = source_records() - changed = source_records() - OriginalSourceMetadata.from_mapping({"well": "A01", "site": "002", "z_index": "000"}).merge_into( - changed, path="/input/A01_z000.tif", - ) + changed = source_records(literal_site="002") assert SourceImageIdentity("/input/A01_z000.tif", original).identity != SourceImageIdentity( "/input/A01_z000.tif", changed, ).identity From 7258826d7198f2bb7010763276dba02320d62142 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 00:54:05 -0400 Subject: [PATCH 118/125] Warm registered configuration source before execution readiness --- external/ObjectState | 2 +- external/python-introspect | 2 +- .../backends/lib_registry/registry_service.py | 11 +++ tests/unit/test_callable_abi_preparation.py | 69 ++++++++++++++++++- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/external/ObjectState b/external/ObjectState index b0004daf32..3831b561ca 160000 --- a/external/ObjectState +++ b/external/ObjectState @@ -1 +1 @@ -Subproject commit b0004daf3223326ac78689ba7b2e68abc18ae298 +Subproject commit 3831b561cafc55a74d97ebde76cf528cf57b055f diff --git a/external/python-introspect b/external/python-introspect index 4e8f8bde5f..d34ad8fd25 160000 --- a/external/python-introspect +++ b/external/python-introspect @@ -1 +1 @@ -Subproject commit 4e8f8bde5f750d9a50eacac80acd3806123addd8 +Subproject commit d34ad8fd251a7fd191847a7b5a920eaded0d98ed diff --git a/openhcs/processing/backends/lib_registry/registry_service.py b/openhcs/processing/backends/lib_registry/registry_service.py index b730021e07..dd327c12a5 100644 --- a/openhcs/processing/backends/lib_registry/registry_service.py +++ b/openhcs/processing/backends/lib_registry/registry_service.py @@ -14,9 +14,12 @@ import threading from collections.abc import Callable from concurrent.futures import CancelledError +from dataclasses import is_dataclass from typing import TYPE_CHECKING, Dict, Optional from arraybridge import MemoryType +from objectstate import LazyDataclassFactory +from python_introspect import SignatureAnalyzer from pyqt_reactive.process_launch import BackgroundProcessLaunchPolicy from zmqruntime import OperationCancellation from zmqruntime.client import endpoint_process @@ -164,6 +167,14 @@ def prepare_in_current_process( emit_status("Resolving prepared callable signatures") for func in callables: CallableProjection.from_callable(func).warm_canonical_signature() + emit_status("Preparing registered configuration source declarations") + for declaration in dict.fromkeys( + declaration + for pair in LazyDataclassFactory.registered_type_pairs() + for declaration in pair + if is_dataclass(declaration) + ): + SignatureAnalyzer.prepare_dataclass_declaration(declaration) emit_status(f"Registered kernels ready ({len(callables)} callables)") emit_status(f"Function catalog ready ({len(cls._metadata_cache)} callables)") return cls._metadata_cache diff --git a/tests/unit/test_callable_abi_preparation.py b/tests/unit/test_callable_abi_preparation.py index dd3f1aa8e9..d7fe5ecddc 100644 --- a/tests/unit/test_callable_abi_preparation.py +++ b/tests/unit/test_callable_abi_preparation.py @@ -3,7 +3,7 @@ import inspect import pickle from types import SimpleNamespace -from dataclasses import replace +from dataclasses import dataclass, field, replace from functools import wraps from unittest.mock import Mock from typing import Any @@ -12,6 +12,8 @@ import numpy as np import pytest from arraybridge import ArrayPayload +import objectstate.lazy_factory as lazy_factory +from python_introspect import SignatureAnalyzer import openhcs.core.callable_contract as contract_module from openhcs.core.callable_contract import ( @@ -165,6 +167,71 @@ def raw(image: np.ndarray): assert second.canonical_parameter_annotations["image"] is ArrayPayload +def test_registry_prepares_current_configuration_declarations_before_ready(monkeypatch): + events = [] + factories = [] + + def default_value(): + factories.append("evaluated") + return 3 + + @dataclass + class PublicConfig: + value: int = field(default_factory=default_value) + + @dataclass + class LazyConfig: + value: int | None = None + + @dataclass + class LaterPublicConfig: + value: int = field(default_factory=default_value) + + @dataclass + class LaterLazyConfig: + value: int | None = None + + def raw(image: np.ndarray): + return image + + metadata = {"raw": SimpleNamespace(func=raw)} + monkeypatch.setattr(RegistryService, "_metadata_cache", metadata) + monkeypatch.setattr(lazy_factory, "_lazy_type_registry", {}) + monkeypatch.setattr(lazy_factory, "_base_to_lazy_registry", {}) + monkeypatch.setattr(PreparationCacheBatch, "populate_child_caches", lambda *a, **k: None) + pairs = iter(((LazyConfig, PublicConfig), (LaterLazyConfig, LaterPublicConfig))) + + def prepare(owner): + events.append("hook") + lazy_factory.register_lazy_type_mapping(*next(pairs)) + + original = SignatureAnalyzer.prepare_dataclass_declaration + + def declaration(config_type): + assert "hook" in events + events.append(config_type) + original(config_type) + + monkeypatch.setattr(CallablePreparation, "prepare", prepare) + monkeypatch.setattr(SignatureAnalyzer, "prepare_dataclass_declaration", declaration) + # The registry admits types that are not dataclasses; readiness filters them. + lazy_factory.register_lazy_type_mapping(dict, list) + assert RegistryService.prepare_in_current_process(status_callback=events.append) is metadata + assert factories == [] + first_ready = next(i for i, item in enumerate(events) if isinstance(item, str) and "kernels ready" in item) + assert events.index(LazyConfig) < first_ready + assert events.index(PublicConfig) < first_ready + assert LaterPublicConfig not in events and dict not in events and list not in events + events.clear() + # An already loaded callable catalog must not freeze the configuration roster. + RegistryService.prepare_in_current_process(status_callback=events.append) + second_ready = next(i for i, item in enumerate(events) if isinstance(item, str) and "kernels ready" in item) + for config_type in (LazyConfig, PublicConfig, LaterLazyConfig, LaterPublicConfig): + assert events.count(config_type) == 1 + assert events.index(config_type) < second_ready + assert factories == [] + + def test_authored_compilation_prepares_before_signature_and_refreshes_each_compilation(monkeypatch): def raw(image: np.ndarray, scale=2): return image From 7070bcef493b546e7ad522e452f36f339dd3eacf Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 01:00:59 -0400 Subject: [PATCH 119/125] Publish ordinary schema-readiness gains and sync merged dependencies --- .../README.rst | 25 ++++ .../recipes/controller.py | 136 ++++++++++++++++++ .../recipes/validate_science.py | 88 ++++++++++++ .../runtime_plumbing_frontier_20261003.rst | 69 ++++++++- external/ObjectState | 2 +- external/python-introspect | 2 +- 6 files changed, 318 insertions(+), 4 deletions(-) create mode 100644 benchmark/results/perf_schema_source_readiness_20261003/README.rst create mode 100644 benchmark/results/perf_schema_source_readiness_20261003/recipes/controller.py create mode 100644 benchmark/results/perf_schema_source_readiness_20261003/recipes/validate_science.py diff --git a/benchmark/results/perf_schema_source_readiness_20261003/README.rst b/benchmark/results/perf_schema_source_readiness_20261003/README.rst new file mode 100644 index 0000000000..17b2e5305c --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/README.rst @@ -0,0 +1,25 @@ +Registered source readiness: ordinary Illumination ABBA +==================================================== + +``receipt.json`` contains public observations, the exact variant hashes and full +science-qualification links. The committed recipes preserve the actual local +experiment; they are evidence, not a turnkey portable benchmark command. Their +paths name the owned checkout and immutable maintenance directory, and the +ordinary helper/bootstrap and physical dataset/native references must already +exist. Do not rerun into a completed output directory. + +Only three source files vary. Their baseline bytes are derived from the recorded +Git revisions in ``variants.json``. Candidate registry bytes are available at +OpenHCS commit ``7258826d7198f2bb7010763276dba02320d62142``; candidate dependency +bytes are available at python-introspect merge +``6f5ac0b79d6cef163df6a8389e72a889a91ed548`` and ObjectState merge +``55e71df39a05aa08d00bc1bb06c2be174ad2c522``. The original maintenance directory +retains all six variant files, source freezes and four immutable output trees. +No dependency source, benchmark images or environments are duplicated here. + +Mean compilation is 0.962152 → 0.587605 seconds; total is +1.693765 → 1.363694 seconds, excluding mandatory server READY/startup and +shutdown. Execution has no demonstrated improvement. All four complete authored +image inventories pass exact byte parity and the retained native physical input +qualification; no new native CP timing is claimed. Other workloads and scaling +remain unmeasured for this change. diff --git a/benchmark/results/perf_schema_source_readiness_20261003/recipes/controller.py b/benchmark/results/perf_schema_source_readiness_20261003/recipes/controller.py new file mode 100644 index 0000000000..effe202bcd --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/recipes/controller.py @@ -0,0 +1,136 @@ +"""Ordinary public ABBA; only three owned source files vary between terminated runs.""" +import csv +import fcntl +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import time + +HERE = Path(__file__).parent +SOURCE = Path('/home/ts/code/projects/openhcs-shared-runtime-plumbing') +MAIN = Path('/home/ts/code/projects/openhcs') +PYTHON = MAIN / '.venv/bin/python' +CASE = 'ExampleIlluminationCorrection_Example3' +VARIANTS = json.loads((HERE / 'variants.json').read_text()) + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + +ordinary = load('ordinary_schema_source', '/tmp/run_integrated_fields_ordinary_abba_20261002.py') +base = load('schema_source_inputs', '/var/tmp/run_narrow450_runtime_phase_probe_20261002.py') +base.SOURCE = SOURCE +base.CASES = (CASE,) +sparse = load('schema_source_sparse', '/var/tmp/openhcs_measurement_sparse_inventory_v2_20261003.py') + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + +def git(repo, *args): + return subprocess.check_output(['git', '-C', str(repo), *args], text=True).strip() + +def environment(): + env = ordinary.environment(SOURCE) + for key in tuple(env): + if key.startswith(('OPENHCS_DIAGNOSTIC_', 'OPENHCS_CAPTURE_CELLPROFILER_FIXTURES_', 'OPENHCS_PROFILE_FUNCTION_RUNTIME')) or key == 'OPENHCS_WORKER_PROFILE_DIR': + del env[key] + env.update(PYTHONDONTWRITEBYTECODE='1', TMPDIR=str(HERE / 'tmp')) + return env + +def sources(): + dependencies = {} + for line in git(SOURCE, 'config', '--file', '.gitmodules', '--get-regexp', 'path$').splitlines(): + _, relative = line.split(maxsplit=1) + repo = SOURCE / relative + dependencies[relative] = {'head': git(repo, 'rev-parse', 'HEAD'), 'status': git(repo, 'status', '--porcelain'), 'files': base.tracked_source_hashes(repo)} + return {'head': git(SOURCE, 'rev-parse', 'HEAD'), 'status': git(SOURCE, 'status', '--porcelain'), 'files': base.tracked_source_hashes(SOURCE), 'dependencies': dependencies, + 'native_binaries': {r: sha(SOURCE/r) for r in ordinary.NATIVE}} + +def select(label): + # This checkout is owned, and the source files are not being edited by agents. + # No live server may remain while its source is changed. + for entry in VARIANTS: + target = Path(entry['path']) + assert target.is_relative_to(SOURCE) + current = sha(target) + assert current in {sha(entry['A']), sha(entry['B'])}, 'Unexpected source edit' + target.write_bytes(Path(entry[label]).read_bytes()) + +def assert_endpoint_terminal(receipt): + import psutil + endpoint = receipt['endpoint_provenance'] + pid = endpoint['endpoint_pid'] + try: + process = psutil.Process(pid) + assert abs(process.create_time() - endpoint['endpoint_create_time_epoch_seconds']) > 0.1 or not process.is_running() or process.status() == psutil.STATUS_ZOMBIE, 'Measured endpoint remains live' + except psutil.NoSuchProcess: + pass + +def main(): + assert shutil.disk_usage('/var/tmp').free >= 512 * 1024**2 + assert shutil.disk_usage('/home').free >= 2 * 1024**3 + assert not (HERE / 'freeze.json').exists() + (HERE / 'tmp').mkdir() + env = environment() + original = sources() + assert not original['status'], 'Commit source before measurement' + physical = base.physical_input_hashes() + sparse_state = sparse.sparse_source_inventory(SOURCE) + installed = subprocess.check_output([str(PYTHON), '-I', '-m', 'pip', 'freeze', '--all'], text=True) + imports = subprocess.check_output([str(PYTHON), '-c', 'import openhcs,objectstate,python_introspect,json;print(json.dumps({m.__name__:m.__file__ for m in (openhcs,objectstate,python_introspect)}))'], env=env, cwd=str(HERE/'tmp'), text=True) + for path in json.loads(imports).values(): + assert Path(path).is_relative_to(SOURCE), path + freeze = {'candidate': original, 'physical_inputs': physical, 'sparse_inventory': sparse_state, 'installed_freeze': installed, 'python_sha256': sha(PYTHON.resolve()), 'controller_sha256': sha(__file__), 'environment': env, 'imports': json.loads(imports), + 'variants': [{**r, 'A_sha256': sha(r['A']), 'B_sha256': sha(r['B'])} for r in VARIANTS], + 'scope': 'Four ordinary uninstrumented public Illumination runs ABBA, CPU5 1w_1t INLINE, default OUTCOMES and memory observer. Server READY preparation/startup and shutdown excluded. A is exact prior source bytes for three files; dependency Git HEADs identify containing checkout, file hashes identify actual baseline contents. Only source lookup, existing registered type projection and registry readiness differ. No scientific processing or numerical changes. No new native CP execution.'} + (HERE/'freeze.json').write_text(json.dumps(freeze, indent=2)+'\n') + rows = [] + lock = Path('/tmp/openhcs-benchmark-xdg-cache/openhcs/official30-runtime.lock') + lock.parent.mkdir(parents=True, exist_ok=True) + try: + with lock.open('a+') as lease: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + for index, label in enumerate(('A', 'B', 'B', 'A')): + select(label) + before = sources() + dest = HERE / f'{index}-{label}' + assert not dest.exists() + command = ['/usr/bin/taskset', '-c', '5', str(PYTHON), '-c', ordinary.BOOTSTRAP, str(SOURCE/'scripts/benchmark_cppipe_well_throughput.py'), '--manifest', str(SOURCE/'benchmark/manifests/official30_portable_axis1.json'), '--mode', '1w_1t', '--case', CASE, '--output-dir', str(dest/'public')] + record = {'index': index, 'label': label, 'source': before, 'command': command} + (HERE/f'{index}-{label}-source.json').write_text(json.dumps(record, indent=2)+'\n') + print(json.dumps({'status': 'RUNNING', 'index': index, 'label': label}), flush=True) + started = time.monotonic() + with (HERE/f'{index}-{label}.log').open('x') as stream: + result = subprocess.run(command, cwd=str(HERE/'tmp'), env=env, stdout=stream, stderr=subprocess.STDOUT, timeout=900) + assert sources() == before, 'Source/dependency/native bytes changed during run' + assert base.physical_input_hashes() == physical + assert subprocess.check_output([str(PYTHON), '-I', '-m', 'pip', 'freeze', '--all'], text=True) == installed + assert sha(__file__) == freeze['controller_sha256'] + measured = list(csv.DictReader((dest/'public/well_throughput.csv').open())) + assert result.returncode == 0 and len(measured) == 1 and measured[0]['status'] == 'success' and measured[0]['successful_wells'] == '1' and measured[0]['execution_route'] == 'ordinary-zmq-outcomes-v1' + receipt_path, = (dest/'public'/CASE/'wells_1/workers_1/ordinary_run_evidence').glob('*/measured_pipeline_receipt.json') + receipt = json.loads(receipt_path.read_text()) + assert receipt['observation_export_scope'] == 'outcomes' and receipt['observed_axis_count'] == receipt['expected_axis_count'] == 1 + assert_endpoint_terminal(receipt) + assert sum(p.stat().st_size for p in dest.rglob('*') if p.is_file()) < 128 * 1024**2, 'Output cap' + rows.append({'label': label, 'directory': str(dest), 'seconds_including_startup_for_audit_only': time.monotonic()-started, 'public_rows': measured, 'receipt': str(receipt_path), 'receipt_sha256': sha(receipt_path), 'source_freeze_sha256': sha(HERE/f'{index}-{label}-source.json')}) + (HERE/'observations.json').write_text(json.dumps({'rows': rows, 'science': 'PENDING', 'scope': freeze['scope']}, indent=2)+'\n') + print(json.dumps({'status': 'RUN_COMPLETE', 'label': label, 'timings': {k: measured[0][k] for k in ('compile_seconds', 'execute_seconds', 'total_seconds')}}), flush=True) + finally: + select('B') + assert sources() == original + sparse.validate_sparse_source_inventory(SOURCE, sparse_state) + from statistics import mean + means = {label: {key: mean(float(row[key]) for r in rows if r['label']==label for row in r['public_rows']) for key in ('compile_seconds', 'execute_seconds', 'total_seconds')} for label in ('A', 'B')} + result = {'means': means, 'saved_seconds': {key: means['A'][key]-means['B'][key] for key in means['A']}, 'rows': rows, 'science': 'PENDING', 'scope': freeze['scope']} + (HERE/'observations.json').write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps({'status': 'ABBA_MEASUREMENT_COMPLETE_SCIENCE_PENDING', 'means': means, 'saved_seconds': result['saved_seconds']}), flush=True) + +if __name__ == '__main__': + main() diff --git a/benchmark/results/perf_schema_source_readiness_20261003/recipes/validate_science.py b/benchmark/results/perf_schema_source_readiness_20261003/recipes/validate_science.py new file mode 100644 index 0000000000..fcbe65f2a6 --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/recipes/validate_science.py @@ -0,0 +1,88 @@ +"""Strict complete authored image inventory and retained native input qualification.""" +import openhcs +from pathlib import Path +import hashlib +import json + +from benchmark.adapters.openhcs import _strict_cellprofiler_runtime_equivalence_policy +from benchmark.matched_cellprofiler_batch import _require_compared_output_inventory +from openhcs.core.equivalence.comparison import runtime_image_differences, runtime_table_differences +from openhcs.core.equivalence.outputs import RuntimeOutputSnapshot +from openhcs.core.runtime_exports import RuntimeExportObservation +from openhcs.core.virtual_workspace_metadata import METADATA_CONFIG, VirtualWorkspaceSourceProjectionEntries +from openhcs.core.source_projection import SourceProjectionSet + +HERE = Path(__file__).parent +REFERENCE = HERE.parent/'illumination-compiler-ledger-v4' +REFERENCE_SCIENCE = HERE.parent/'illumination-compiler-ledger-v4-science.json' +NATIVE = Path('/var/tmp/openhcs-current-illumination3-native-timing-science-v1-20261002.json') +CASE = 'ExampleIlluminationCorrection_Example3' + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + +def read(directory): + receipt_path, = (directory/'public'/CASE/'wells_1/workers_1/ordinary_run_evidence').glob('*/measured_pipeline_receipt.json') + receipt = json.loads(receipt_path.read_text()) + assert receipt['observation_export_scope'] == 'outcomes' + assert receipt['observed_axis_count'] == receipt['expected_axis_count'] == 1 + assert sha(receipt['observation_export_path']) == receipt['observation_export_sha256'] + assert sha(receipt['results_summary_path']) == receipt['results_summary_sha256'] + root, = map(Path, receipt['output_roots']) + assert root.resolve().is_relative_to(directory.resolve()) + files = frozenset(p for p in root.rglob('*') if p.is_file()) + exports = RuntimeExportObservation.from_output_roots((root,)) + snapshot = RuntimeOutputSnapshot.from_export_observation(exports, source_workspaces=(root,)) + return receipt_path, receipt, root, files, exports, snapshot + +def main(): + assert not (HERE/'science.json').exists() + obs = json.loads((HERE/'observations.json').read_text()) + assert len(obs['rows']) == 4 and [r['label'] for r in obs['rows']] == ['A','B','B','A'] + qualification = json.loads(REFERENCE_SCIENCE.read_text()) + assert qualification['status'] == 'CURRENT_BB9_DIAGNOSTIC_FULL_AUTHORED_IMAGE_INVENTORY_BYTE_NATIVE_INPUT_TRANSITIVE_PASS' + native = json.loads(NATIVE.read_text()) + assert native['status'] == 'PASS_AUTHORED_NONEMPTY_SAVED_IMAGE_SCIENCE_AND_MATCHED_FRESH_NATIVE_TIMING' + for role in ('native_report','source_environment_input_witness','saved_image_science_receipt'): + assert sha(native[role]) == native[role+'_sha256'] + ref = read(REFERENCE) + assert sha(ref[0]) == qualification['current_receipt_sha256'] + assert {str(p.relative_to(ref[2])): sha(p) for p in ref[3] if p.suffix == '.npy'} == qualification['scientific_npy_bytes'] + before = {str(p): sha(p) for p in ref[3]} + policy = _strict_cellprofiler_runtime_equivalence_policy() + results = [] + freeze = json.loads((HERE/'freeze.json').read_text()) + cpp = freeze['physical_inputs']['cppipe_path'] + assert cpp['path'] == native['source_cppipe'] and list(cpp['files'].values()) == [native['source_cppipe_sha256']] + environments = [] + for run in obs['rows']: + actual = read(Path(run['directory'])) + assert sha(actual[0]) == run['receipt_sha256'] + before.update({str(p): sha(p) for p in actual[3]}) + td = runtime_table_differences(ref[5].tables, actual[5].tables, policy) + im = runtime_image_differences(ref[5].images, actual[5].images, policy) + assert not td and not im, (td, im) + assert not actual[5].tables and len(actual[5].images) == len(ref[5].images) == 2 + _require_compared_output_inventory(reference_files=ref[3]-frozenset(METADATA_CONFIG.managed_paths(ref[2])), candidate_files=actual[3], reference_exports=ref[4], candidate_exports=actual[4], reference_snapshot=ref[5], candidate_snapshot=actual[5], candidate_managed_files=frozenset(METADATA_CONFIG.managed_paths(actual[2]))) + images = {str(p.relative_to(actual[2])): sha(p) for p in actual[3] if p.suffix == '.npy'} + assert images == qualification['scientific_npy_bytes'] and len(images) == 2 + workspace = Path(actual[1]['execution_plate_id']) + subdirs = json.loads((workspace/'openhcs_metadata.json').read_text())['subdirectories'] + entries = tuple(entry for s in subdirs.values() for entry in VirtualWorkspaceSourceProjectionEntries.from_subdirectory(s).entries.values()) + projections = SourceProjectionSet(entries) + assert len(projections.plane_projections) == 1 and not projections.artifact_projections + assert all(not p.ref.source_axis_indices for p in projections.plane_projections) + selected = {str(Path(p.ref.backend_address).resolve(strict=True)): sha(p.ref.backend_address) for p in projections.plane_projections} + assert selected == native['physical_source_sha256'] + assert actual[1]['global_config_source_sha256'] == ref[1]['global_config_source_sha256'] + assert actual[1]['endpoint_provenance']['client_openhcs_file'] == str(Path(freeze['imports']['openhcs'])) + environments.append(actual[1]['server_environment']) + results.append({'label': run['label'], 'directory':run['directory'], 'receipt_sha256':sha(actual[0]), 'scientific_npy_bytes':images, 'physical_selected_inputs':selected, 'image_differences':im, 'table_differences':td, 'full_physical_inventory':'PASS'}) + assert all(e == environments[0] for e in environments) + assert all(sha(p) == digest for p,digest in before.items()) + result = {'status':'ALL4_ORDINARY_AUTHORED_IMAGES_EXACT_AND_RETAINED_NATIVE_INPUT_PARITY_PASS', 'runs':results, 'means':obs['means'], 'saved_seconds':obs['saved_seconds'], 'all_before_after_output_sha256':before, 'server_environment':environments[0], 'freeze_sha256':sha(HERE/'freeze.json'), 'reference_science_sha256':sha(REFERENCE_SCIENCE), 'native_receipt_sha256':sha(NATIVE), 'validator_sha256':sha(__file__), 'limits':['Two observations per variant; no execution speedup is claimed.', 'Authored workload exports two images and no tables.', 'Native comparison is retained matched CP4.2.8.1 evidence, not a fresh native run.', 'Ordinary pipeline total excludes mandatory server startup and shutdown.']} + (HERE/'science.json').write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps({'status':result['status'], 'means':obs['means'], 'saved_seconds':obs['saved_seconds'], 'science_sha256':sha(HERE/'science.json')})) + +if __name__ == '__main__': + main() diff --git a/docs/plans/runtime_plumbing_frontier_20261003.rst b/docs/plans/runtime_plumbing_frontier_20261003.rst index a41e346ec9..6476dd2d31 100644 --- a/docs/plans/runtime_plumbing_frontier_20261003.rst +++ b/docs/plans/runtime_plumbing_frontier_20261003.rst @@ -284,5 +284,70 @@ Local source-qualified receipts: * ``/home/ts/.local/state/openhcs-maintenance/20261003/illumination-schema-cache-topology-v4-20261003.json`` (SHA-256 ``134244ea39004ccfc86d20fa14d4fdc49e78aa72f4a92349092c6f420c1b89e3``). -No production optimization, new native repetition, original R0/R1 waiver, -full-catalog/scaling completion or accepted ordinary speedup is claimed here. +The preceding V3/V4 diagnostics are attribution only. They do not establish a +production saving, new native repetition, original R0/R1 waiver or +full-catalog/scaling completion. + +Registered configuration source readiness +----------------------------------------- + +The decisive source-only prototype found repeated Python 3.10--3.12 class +source lookup walking the complete config module. The existing +``SignatureAnalyzer`` now derives bounded module AST and qualified-class source +views from current immutable source contents. Python 3.13 and newer retain the +native locator. Live annotations, factories, mutable field/inherited docs, +loader invalidation and the original completed value-analysis cache remain at +their existing observation points. Earlier class-identity snapshots failed +real mutation controls and were rejected. + +``LazyDataclassFactory.registered_type_pairs()`` projects the existing registry +without a second roster or constructing values. ``RegistryService`` prepares +both public and proxy declarations after callable hooks and before READY, +including declarations registered after the catalog was cached. Default +factories are not evaluated during source preparation. + +At OpenHCS ``7258826d7198f2bb7010763276dba02320d62142``, four ordinary public +Illumination runs in ABBA order measured: + +.. list-table:: Ordinary 1w_1t mean seconds; two observations per variant + :header-rows: 1 + + * - Phase + - Prior source + - Prepared source + * - Compilation + - 0.962152 + - 0.587605 + * - Execution job + - 0.382655 + - 0.400589 + * - Total pipeline + - 1.693765 + - 1.363694 + +Compilation saves 0.374547s (38.9%); total saves 0.330070s (19.5%). No execution +gain is claimed. All four runs use CPU5, INLINE one-worker execution, normal +OUTCOMES and the default memory observer, without hooks or profilers. Mandatory +server startup/preparation and shutdown are outside total pipeline timing. +Exactly three owned source files vary; immutable before/after hashes identify +actual baseline contents separately from containing dependency Git HEADs. +All source, dependencies, native binaries, environment and physical inputs were +checked before/after each run; the clean candidate was restored afterward. + +All four complete authored output inventories retain both NPY artifacts exactly +and match the qualified physical TIFF/pipeline/native evidence. The workload +authors no tables. Native CellProfiler was not rerun in this experiment; broad +scaling or new native speedup claims are not made. Science receipt SHA-256: +``918c36612ec20738ad2d2760da85c9e9be07e81d4cd4bf0f60ec91dfcf906488``. + +The 136 upstream, 25 existing lazy-factory and 119 OpenHCS readiness/catalog/ +startup tests pass. python-introspect PR #6 and ObjectState PR #9 are merged and +formally close their issues #5 and #8. Their merge commits change no measured +source bytes. OpenHCS wiring is pushed in draft #394; a small main-only readiness +change can be isolated without merging the unresolved broad branch. +Reviewable measurement receipt and recipes are under +``benchmark/results/perf_schema_source_readiness_20261003``; original immutable +outputs/freeze remain under the corresponding maintenance ABBA directory. +Runtime plumbing, full-catalog/scaling measurements, fresh native repetitions +and figures remain outstanding. Original whole-branch R0/R1 failures remain +explicit and unwaived. diff --git a/external/ObjectState b/external/ObjectState index 3831b561ca..55e71df39a 160000 --- a/external/ObjectState +++ b/external/ObjectState @@ -1 +1 @@ -Subproject commit 3831b561cafc55a74d97ebde76cf528cf57b055f +Subproject commit 55e71df39a05aa08d00bc1bb06c2be174ad2c522 diff --git a/external/python-introspect b/external/python-introspect index d34ad8fd25..6f5ac0b79d 160000 --- a/external/python-introspect +++ b/external/python-introspect @@ -1 +1 @@ -Subproject commit d34ad8fd251a7fd191847a7b5a920eaded0d98ed +Subproject commit 6f5ac0b79d6cef163df6a8389e72a889a91ed548 From 986f6a917ecfed38f121158201ea902e091d2992 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 01:01:15 -0400 Subject: [PATCH 120/125] Retain source-readiness measurement receipt for review --- .../receipt.json | 261 ++++++++++++++++++ .../recipes/variants.json | 26 ++ 2 files changed, 287 insertions(+) create mode 100644 benchmark/results/perf_schema_source_readiness_20261003/receipt.json create mode 100644 benchmark/results/perf_schema_source_readiness_20261003/recipes/variants.json diff --git a/benchmark/results/perf_schema_source_readiness_20261003/receipt.json b/benchmark/results/perf_schema_source_readiness_20261003/receipt.json new file mode 100644 index 0000000000..817e85e994 --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/receipt.json @@ -0,0 +1,261 @@ +{ + "status": "ALL4_ORDINARY_AUTHORED_IMAGES_EXACT_AND_RETAINED_NATIVE_INPUT_PARITY_PASS", + "case": "ExampleIlluminationCorrection_Example3", + "mode": "1w_1t", + "cpu": 5, + "candidate_openhcs_commit": "7258826d7198f2bb7010763276dba02320d62142", + "variants": [ + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/openhcs/processing/backends/lib_registry/registry_service.py", + "baseline_revision": "4eaacf17e5de3dba59ec217f133fa6e1821c8641", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing", + "relative": "openhcs/processing/backends/lib_registry/registry_service.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-B.source", + "A_sha256": "73e0d41f98d7ea5bb3fe076b97b97d9980b621c54f2d23b808550e00bae481d4", + "B_sha256": "d9e6e30b627fae7b01e08872c58ad26ee060919b6b6bd9fd9174223f2ec5a95b" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect/src/python_introspect/signature_analyzer.py", + "baseline_revision": "13f6ff3b43aaa430f5e771848f08d99faf941158", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect", + "relative": "src/python_introspect/signature_analyzer.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-B.source", + "A_sha256": "51f4729861062ab857153c57d64b01bd7b0f47daa6f18fc4443819fb75c77b45", + "B_sha256": "7e25a0355159c86d361a21a7afc449ec758cbf8a32e9b72d06de66a924f04159" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState/src/objectstate/lazy_factory.py", + "baseline_revision": "b0004daf3223326ac78689ba7b2e68abc18ae298", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState", + "relative": "src/objectstate/lazy_factory.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-B.source", + "A_sha256": "f81ce722695dbab98fd02658f2015e5cb93c1e1e69547d24249e62d896e2a828", + "B_sha256": "90637bdad398bae1348168e62533d69618ec965341b1b27057292e709c2a1f85" + } + ], + "means": { + "A": { + "compile_seconds": 0.9621515274047852, + "execute_seconds": 0.38265538215637207, + "total_seconds": 1.693764767376706 + }, + "B": { + "compile_seconds": 0.5876049995422363, + "execute_seconds": 0.40058863162994385, + "total_seconds": 1.3636944827158004 + } + }, + "saved_seconds": { + "compile_seconds": 0.37454652786254883, + "execute_seconds": -0.017933249473571777, + "total_seconds": 0.3300702846609056 + }, + "runs": [ + { + "label": "A", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.9799041748046875", + "prepare_seconds": "0.0", + "execute_seconds": "0.38725757598876953", + "total_seconds": "1.7173671890050173", + "wells_per_second": "2.5822606502835725", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1011.84765625", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "a37695e774a71b04c3e2a390df76245c973458380fca25a9aac513dbf5e8be8e" + } + ], + "receipt_sha256": "6805ee89e2872aea6199e959e53b956ef144e2ff82516fbf6ac68042450d6fd2", + "source_freeze_sha256": "d71dcaef08a9eb2ac12a00b5e004aedfcd9b9606d860e6a8bca5da73baa9900b" + }, + { + "label": "B", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.593247652053833", + "prepare_seconds": "0.0", + "execute_seconds": "0.42087531089782715", + "total_seconds": "1.4185294113121927", + "wells_per_second": "2.3760006208650304", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1043.54296875", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "3c135c9fa72e1a8ec100dca796e96c7f6721cccfab82316b3836236b6b4e75b0" + } + ], + "receipt_sha256": "c8f76ce4b6acaa60f5bbfe13c5a5ca924e299d5c8834e249f757dd53adc434c7", + "source_freeze_sha256": "ac0a8d0ad5938927dbddf01b937ef15387a15c6701d87d292ed486fbcbec7870" + }, + { + "label": "B", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.5819623470306396", + "prepare_seconds": "0.0", + "execute_seconds": "0.38030195236206055", + "total_seconds": "1.3088595541194081", + "wells_per_second": "2.629489524807818", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1043.19140625", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "3c135c9fa72e1a8ec100dca796e96c7f6721cccfab82316b3836236b6b4e75b0" + } + ], + "receipt_sha256": "8a7f0ecb3274b47312e8ee1431b4bd75f9aaba713fd76d3e9ec28ab038aa93da", + "source_freeze_sha256": "d6608b629bb37ecb89d30797258d674d816857671e77fcf580bec3d62910a7a2" + }, + { + "label": "A", + "public_rows": [ + { + "case_name": "ExampleIlluminationCorrection_Example3", + "mode_name": "1w_1t", + "worker_count": "1", + "well_count": "1", + "compile_seconds": "0.9443988800048828", + "prepare_seconds": "0.0", + "execute_seconds": "0.3780531883239746", + "total_seconds": "1.6701623457483947", + "wells_per_second": "2.6451304495891326", + "successful_wells": "1", + "native_single_sample_execution_seconds": "", + "projected_native_execution_seconds": "", + "projected_execution_speedup": "", + "peak_memory_mb": "1012.59375", + "status": "success", + "memory_limit_mb": "", + "error_message": "", + "execution_route": "ordinary-zmq-outcomes-v1", + "server_lifecycle": "reused-per-sweep", + "run_input_sha256": "a37695e774a71b04c3e2a390df76245c973458380fca25a9aac513dbf5e8be8e" + } + ], + "receipt_sha256": "4a804982debefcd6cd075ac16520e85036281b3eddd78cabaf86f41f2841b4c6", + "source_freeze_sha256": "6b40a27bf359bb67febc0af967221829a7719cac0d965704609c0d36991f608e" + } + ], + "tests": { + "python_introspect": 136, + "objectstate_lazy_factory": 25, + "openhcs_readiness": 119 + }, + "upstream_prs": [ + "https://github.com/OpenHCSDev/python-introspect/pull/6", + "https://github.com/OpenHCSDev/ObjectState/pull/9" + ], + "science_runs": [ + { + "label": "A", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-A", + "receipt_sha256": "6805ee89e2872aea6199e959e53b956ef144e2ff82516fbf6ac68042450d6fd2", + "scientific_npy_bytes": { + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6", + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + }, + { + "label": "B", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-B", + "receipt_sha256": "c8f76ce4b6acaa60f5bbfe13c5a5ca924e299d5c8834e249f757dd53adc434c7", + "scientific_npy_bytes": { + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a", + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + }, + { + "label": "B", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-B", + "receipt_sha256": "8a7f0ecb3274b47312e8ee1431b4bd75f9aaba713fd76d3e9ec28ab038aa93da", + "scientific_npy_bytes": { + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6", + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + }, + { + "label": "A", + "directory": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/3-A", + "receipt_sha256": "4a804982debefcd6cd075ac16520e85036281b3eddd78cabaf86f41f2841b4c6", + "scientific_npy_bytes": { + "results/reference_image__ConvexHullCorrWorm.npy": "9ba1259e4ca007c06f14be6946e32a86307ec2dfb9952b73fa2cc253ccae8b3a", + "results/reference_image__PolynomialCorrected.npy": "8875da63a072c8f2de4f5cb707d779a064b663bf69aa0f304fe6fa74d4ff57a6" + }, + "physical_selected_inputs": { + "/home/ts/.cache/openhcs/cellprofiler_examples/ExampleIlluminationCorrection/images/ADSAStaphInfection2_A01_w2247376DD-6ADD-442D-AE47-F54A05F3EA94.tif": "5a57b7557bb0da5e241b48851b82d936c7ec36ede7aaca5e77b28ac4bfebf502" + }, + "image_differences": [], + "table_differences": [], + "full_physical_inventory": "PASS" + } + ], + "scope": "Four ordinary uninstrumented public Illumination runs ABBA, CPU5 1w_1t INLINE, default OUTCOMES and memory observer. Server READY preparation/startup and shutdown excluded. A is exact prior source bytes for three files; dependency Git HEADs identify containing checkout, file hashes identify actual baseline contents. Only source lookup, existing registered type projection and registry readiness differ. No scientific processing or numerical changes. No new native CP execution.", + "limits": [ + "Two observations per variant; no execution speedup is claimed.", + "Authored workload exports two images and no tables.", + "Native comparison is retained matched CP4.2.8.1 evidence, not a fresh native run.", + "Ordinary pipeline total excludes mandatory server startup and shutdown." + ], + "artifact_hashes": { + "freeze.json": "2c5d9272f1693e1da821ad45c6db3190cd046f12f446142f9c263611f8a81da8", + "observations.json": "aa72dc885608831f3bbc21efc62f2739d86ec74d64f5ae56d0ee27cd129792fd", + "science.json": "918c36612ec20738ad2d2760da85c9e9be07e81d4cd4bf0f60ec91dfcf906488", + "controller.py": "6fcf1cb1008ac32ee3ceaff8fd61f6db1fdba87544e1ffee1e4b3d484ce30b90", + "validate_science.py": "86fd34f0ca552309a453c5c169a97b949304aa37fb642a66972f4d12f9940684" + }, + "authoritative_artifacts": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1", + "remaining_work": "No execution speedup claimed. Runtime plumbing, other workloads, fresh native/scaling sweeps and figures remain outstanding." +} diff --git a/benchmark/results/perf_schema_source_readiness_20261003/recipes/variants.json b/benchmark/results/perf_schema_source_readiness_20261003/recipes/variants.json new file mode 100644 index 0000000000..3cbd1f0ed4 --- /dev/null +++ b/benchmark/results/perf_schema_source_readiness_20261003/recipes/variants.json @@ -0,0 +1,26 @@ +[ + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/openhcs/processing/backends/lib_registry/registry_service.py", + "baseline_revision": "4eaacf17e5de3dba59ec217f133fa6e1821c8641", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing", + "relative": "openhcs/processing/backends/lib_registry/registry_service.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/0-B.source" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect/src/python_introspect/signature_analyzer.py", + "baseline_revision": "13f6ff3b43aaa430f5e771848f08d99faf941158", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/python-introspect", + "relative": "src/python_introspect/signature_analyzer.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/1-B.source" + }, + { + "path": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState/src/objectstate/lazy_factory.py", + "baseline_revision": "b0004daf3223326ac78689ba7b2e68abc18ae298", + "repository": "/home/ts/code/projects/openhcs-shared-runtime-plumbing/external/ObjectState", + "relative": "src/objectstate/lazy_factory.py", + "A": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-A.source", + "B": "/home/ts/.local/state/openhcs-maintenance/20261003/schema-source-readiness-ordinary-abba-v1/2-B.source" + } +] From 8e06a382e2d4b6d9d9b8ceb93c229717c328d538 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 01:03:41 -0400 Subject: [PATCH 121/125] Sync ObjectState readiness annotation correction from main --- external/ObjectState | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/ObjectState b/external/ObjectState index 55e71df39a..9c31e4b07b 160000 --- a/external/ObjectState +++ b/external/ObjectState @@ -1 +1 @@ -Subproject commit 55e71df39a05aa08d00bc1bb06c2be174ad2c522 +Subproject commit 9c31e4b07b5537c0ed2307c5a63034c708b86bbb From b312148855b6ed486442d15e9395a09e453d77c6 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 01:29:48 -0400 Subject: [PATCH 122/125] Derive provenance fingerprints from canonical metadata fields --- .../runtime_plumbing_frontier_20261003.rst | 34 +++++++++++++ openhcs/core/source_metadata.py | 4 +- tests/unit/test_source_metadata_owner.py | 50 +++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/docs/plans/runtime_plumbing_frontier_20261003.rst b/docs/plans/runtime_plumbing_frontier_20261003.rst index 6476dd2d31..3cbe1622bd 100644 --- a/docs/plans/runtime_plumbing_frontier_20261003.rst +++ b/docs/plans/runtime_plumbing_frontier_20261003.rst @@ -351,3 +351,37 @@ outputs/freeze remain under the corresponding maintenance ABBA directory. Runtime plumbing, full-catalog/scaling measurements, fresh native repetitions and figures remain outstanding. Original whole-branch R0/R1 failures remain explicit and unwaived. + +Canonical provenance identity integration +------------------------------------------ + +Main readiness #514 is merged into this branch. The overlapping readiness +regression was removed from the callable ABI suite; the dedicated main test +remains authoritative. Callable canonical-signature preparation still precedes +registered declaration preparation and kernel readiness. + +The existing ``SourceMetadataFields.provenance_identity_items`` now derives its +representation from ``identity_items``. Equal nested field mappings have equal +provenance fingerprints independent of insertion order. Raw wire order, real +value differences, ordered planes and lifetime-selected views remain intact. +Existing birth fingerprints are preserved by pickle/cloudpickle; they are not +recomputed from subsequently mutated public fields. Historical nested-mapping +fingerprints can therefore differ from newly canonical fingerprints. A fresh +same-source capture is required before candidate fingerprint replay. + +The integrated metadata/transport/readiness controls pass 86 tests in 2.40s. +The separate exact PR #494 source qualification passes its unchanged 31 writer +controls plus seven identity controls. Those writer controls cannot collect on +this branch because its independent ``PointROIOutput`` producer is absent; no +installed, native or live viewer acceptance is inferred. The integrated receipt +is ``~/.local/state/openhcs-maintenance/20261003/pr494-identity-owner-scratch/ +integrated-owned-qualification-v1.json``. + +V8's original producer and downstream loader replay now pass complete value, +metadata, manifest and array-alias gates, and the full six-CSV/120-TIFF science +gate passes. Serialization still drops nested metadata views and source-context +derived caches. These scoped clocks are not complete production-warm performance +evidence. The saved downstream cache hit constructs no new provenance objects; +it does not establish the payoff of a broader loading/recording/publication fix. +The next capture must retain actual nested caches in the same graph, under the +existing admission bounds, without rewarming or inventing state. diff --git a/openhcs/core/source_metadata.py b/openhcs/core/source_metadata.py index c8ab6df14b..06fc651d4a 100644 --- a/openhcs/core/source_metadata.py +++ b/openhcs/core/source_metadata.py @@ -205,11 +205,11 @@ def project() -> SourceMetadataIdentityItems: def provenance_identity_items( cls, metadata: SourceMetadataMapping ) -> tuple[tuple[str, str], ...]: - """Preserve the exact stored repr, including nested mapping insertion order.""" + """Derive stable provenance identity from the owner's canonical field view.""" return cls._view( metadata, "provenance_identity_items", - lambda: tuple(sorted((str(k), repr(v)) for k, v in metadata.items())), + lambda: tuple((key, repr(value)) for key, value in cls.identity_items(metadata)), ) @classmethod diff --git a/tests/unit/test_source_metadata_owner.py b/tests/unit/test_source_metadata_owner.py index 4a492b864d..f204c09c61 100644 --- a/tests/unit/test_source_metadata_owner.py +++ b/tests/unit/test_source_metadata_owner.py @@ -24,6 +24,8 @@ from openhcs.core.source_image_provenance import ( SourceImageIdentity, SourceImageProvenance, + SourceImageProvenancePlanes, + SourceImageProvenance, SourceImageProvenancePlaneRecord, SourcePlaneIndexedMetadata, source_component_metadata_consensus, @@ -271,6 +273,54 @@ def __getattribute__(self, name): assert left != right and left.identity == captured +def test_nested_mapping_order_does_not_change_provenance_identity_or_wire_order(): + raw = {"well": "A01", ORIGINAL_SOURCE_METADATA_FIELD: {"site": "001", "channel": "1"}} + reordered = {ORIGINAL_SOURCE_METADATA_FIELD: {"channel": "1", "site": "001"}, "well": "A01"} + owned = DurableSourceMetadata.from_mapping(reordered) + before = to_jsonable(owned) + original = SourceImageProvenance(source_path="source.tif", source_component_metadata=raw) + for metadata in (reordered, owned): + candidate = SourceImageProvenance(source_path="source.tif", source_component_metadata=metadata) + assert candidate == original + assert candidate.equality_identity == original.equality_identity + assert SourceMetadataFields.provenance_identity_items(raw) == SourceMetadataFields.provenance_identity_items(owned) + assert to_jsonable(owned) == before + assert tuple(reordered) == (ORIGINAL_SOURCE_METADATA_FIELD, "well") + assert tuple(reordered[ORIGINAL_SOURCE_METADATA_FIELD]) == ("channel", "site") + reordered[ORIGINAL_SOURCE_METADATA_FIELD]["site"] = "002" + changed = SourceImageProvenance(source_path="source.tif", source_component_metadata=reordered) + assert changed.equality_identity != original.equality_identity + + +def test_canonical_metadata_identity_preserves_ordered_provenance_planes(): + paths = ("a.tif", "b.tif") + metadata = ({"site": "1"}, {"site": "2"}) + forward = SourceImageProvenancePlanes.from_components(paths=paths, component_metadata=metadata) + reverse = SourceImageProvenancePlanes.from_components( + paths=tuple(reversed(paths)), component_metadata=tuple(reversed(metadata)), + ) + assert SourceImageProvenance(source_image_provenance_planes=forward).equality_identity != SourceImageProvenance( + source_image_provenance_planes=reverse, + ).equality_identity + + +@pytest.mark.parametrize("serializer", (pickle, cloudpickle)) +def test_identity_transport_preserves_birth_fingerprint_after_current_metadata_changes(serializer): + from openhcs.core.orchestrator.execution_result import RuntimeExecutionTransportSerialization + + RuntimeExecutionTransportSerialization.register() + nested = {"site": "001", "channel": "1"} + identity = SourceImageIdentity("original.tif", {ORIGINAL_SOURCE_METADATA_FIELD: nested}) + birth = identity.identity + nested["site"] = "002" + identity.path = "current.tif" + restored = serializer.loads(serializer.dumps(identity)) + assert restored.path == "current.tif" + assert restored.component_metadata[ORIGINAL_SOURCE_METADATA_FIELD]["site"] == "002" + assert restored.identity == birth + assert SourceImageIdentity(restored.path, restored.component_metadata).identity != birth + + def test_raw_composition_snapshots_once_before_consensus_reads(): class ObservableMapping(Mapping): def __init__(self, values): From 67c6acccaa7769922f834b6a5ceaa88725a54980 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 03:17:34 -0400 Subject: [PATCH 123/125] fix(viewer): retain point source extent and calibrated reopen payload --- ...live-fractional-z-point-projection-152.rst | 42 +++++++++ openhcs/core/roi_source_metadata.py | 16 ++-- openhcs/core/viewer_streaming_service.py | 2 +- openhcs/runtime/napari_streaming_handlers.py | 42 +++++++-- openhcs/runtime/napari_viewer_server.py | 21 +++-- .../test_live_fractional_z_point_source.py | 2 +- tests/unit/test_napari_streaming_handlers.py | 86 +++++++++++++++++++ tests/unit/test_streaming_service.py | 6 +- 8 files changed, 185 insertions(+), 32 deletions(-) diff --git a/docs/validation/live-fractional-z-point-projection-152.rst b/docs/validation/live-fractional-z-point-projection-152.rst index 993b17d2b1..37775f781a 100644 --- a/docs/validation/live-fractional-z-point-projection-152.rst +++ b/docs/validation/live-fractional-z-point-projection-152.rst @@ -6,6 +6,48 @@ No competing production claim found:468 changes tracking.py;404 carries saved writer proposals/evidence;394 does not change Napari/viewer sources. Shared materialization source ownership must be coordinated before editing that seam. +Newest public95 checkpoint: FAILED, not accepted +----------------------------------------------- + +The actual public tiny producer compiled, but execution +9292bfb8-4d2e-4e54-9bcc-5af998083fe5 failed at persisted-image metadata +publication after the point archive was written/streamed. Root435/394 owns +that publication join; neither actual conflicting metadata pair is logged. +The original failure, partial live/reopen journals and targets remain immutable. + +Original PolyStore readback of the actual989B ZIP proves all four source planes, +fractional Z1.5, object7 and explicit2/.65/.65 micrometer calibration intact. +Archive SHA256fd0a372f3100c73da9533b308d44c2f85fa7a86b733e2608ae986135cb4e401a. +Persisted reopening transported geometry without its full source record; +the native point handler narrowed represented Z0..3 to occupied0..2; +native presentation calibrated XY but discarded explicit Z spacing. All9 +original XY/XZ/YZ PNGs were personally opened: rendered transport works, +but automatic completion, full source preservation and calibrated acceptance do +not. No scientist input, installed target or native endpoint was changed here. + +This source checkpoint repairs those disjoint receiving owners. StreamingService +keeps the original source-bound ROI payload. ROIArchiveSourceMetadata owns the +single source-only feature exclusion used by native Points and Shapes; it does +not strip transport source facts. Points keeps the full represented source span +from its original anchor while preserving missing-axis/anchor/fractional guards. +NapariAxisPresentation derives scale, units and world translation together for +Image/Shapes/Points and shared-axis reprojection. Explicit3D spacing calibrates Z; +2D/unspecified calibration does not invent it, and selectors/bands stay unscaled. +No mirrored source domain, codec, registry, publication guard relaxation or +shared Root producer/compiler change. Singer522 retirement methods are untouched. + +Before-edit named AST family:704 production+95 original paired dependency modules, +zero parse omissions; source relationships read semantically, not a global NRA +proof. Original source controls incorrectly asserted occupied0..2; the actual +receiving failure demonstrated their insufficient extent assertion. New source +controls require the complete represented extent and actual reopened metadata. +Calibration controls cover2D/3D/relative/unspecified coordinates, scaled offsets, +native Image/Points world alignment and semantic navigation without a Qt app. +Qualification is pending; no installed or new native acceptance is claimed. +The initial direct-source pytest attempt failed before collection because +vendored dirty ZMQ lacks AckReturnRoute. That failure is retained, not fixed by +editing dependency checkouts. + Reported technical failure -------------------------- diff --git a/openhcs/core/roi_source_metadata.py b/openhcs/core/roi_source_metadata.py index 6455aaadf5..f6d90351fc 100644 --- a/openhcs/core/roi_source_metadata.py +++ b/openhcs/core/roi_source_metadata.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import replace from typing import ClassVar @@ -40,13 +40,11 @@ def decode(cls, rois: Sequence[ROI]) -> ImagePayloadMetadata | None: def geometry(cls, rois: Sequence[ROI]) -> list[ROI]: """Keep transport-only source declarations out of the ROI feature table.""" return [ - replace( - roi, - metadata={ - key: value - for key, value in roi.metadata.items() - if key != cls.FIELD - }, - ) + replace(roi, metadata=cls.feature_metadata(roi.metadata)) for roi in rois ] + + @classmethod + def feature_metadata(cls, metadata: Mapping[str, object]) -> dict[str, object]: + """Project features without discarding the transported source record.""" + return {key: value for key, value in metadata.items() if key != cls.FIELD} diff --git a/openhcs/core/viewer_streaming_service.py b/openhcs/core/viewer_streaming_service.py index d878d3146c..89f80d44e2 100644 --- a/openhcs/core/viewer_streaming_service.py +++ b/openhcs/core/viewer_streaming_service.py @@ -846,7 +846,7 @@ def stream_rois( point_domain = ROIFractionalZ.source_component_domain(rois, metadata) if point_domain is not None: point_domains[filename] = point_domain - data_list.append(ROIArchiveSourceMetadata.geometry(rois)) + data_list.append(rois) paths.append(filename) loaded_indices.append(i - 1) diff --git a/openhcs/runtime/napari_streaming_handlers.py b/openhcs/runtime/napari_streaming_handlers.py index 43df62396e..2c7b7e7338 100644 --- a/openhcs/runtime/napari_streaming_handlers.py +++ b/openhcs/runtime/napari_streaming_handlers.py @@ -20,6 +20,7 @@ from openhcs.constants import AllComponents from openhcs.core.artifacts import ObjectArtifactSubjectBinding from openhcs.core.config import NapariDisplayConfig +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, ) @@ -1440,10 +1441,12 @@ def align_coordinates(self, coordinates: np.ndarray) -> np.ndarray: def translate( self, payload_axis_labels: tuple[str, ...] = (), + *, + scale: Sequence[float] | None = None, ) -> tuple[float, ...]: - """Return translation in the same declaration-owned slots as layer data.""" + """Place route offsets in the same native world units as layer scale.""" - return ( + offsets = ( *( float(self.axis_offset(index)) for index in range(len(self.display_axis_components)) @@ -1452,16 +1455,21 @@ def translate( 0.0, 0.0, ) + if scale is None: + return offsets + return tuple( + offset * spacing for offset, spacing in zip(offsets, scale, strict=True) + ) def spatial_layer_kwargs( self, items: Sequence[NapariStreamLayerItem], payload_axis_labels: tuple[str, ...] = (), ) -> dict[str, LayerKwargValue]: - """Project calibrated XY onto aligned native axes for every layer kind. + """Project declared spatial calibration onto every aligned layer kind. - Component and internal payload axes remain dimensionless. Two-dimensional - acquisition calibration never invents physical Z or a color-band axis. + Only a real three-axis source declaration calibrates projected Z. + Acquisition selectors and internal payload bands remain dimensionless. """ spacing = CommonRuntimeValue.from_values( item.image_metadata.source_voxel_spacing for item in items @@ -1470,11 +1478,27 @@ def spatial_layer_kwargs( raise ValueError( "A native viewer route requires consistent source voxel spacing." ) - prefix = len(self.display_axis_components) + len(payload_axis_labels) + component_scales = [1.0] * len(self.display_axis_components) + component_units = ["dimensionless"] * len(self.display_axis_components) + z_component = AllComponents.Z_INDEX.value + if ( + len(spacing.values_zyx) == 3 + and z_component in self.display_axis_components + ): + z_axis = self.display_axis_components.index(z_component) + component_scales[z_axis] = spacing.spacing_for_ndim(3)[0] + component_units[z_axis] = spacing.native_coordinate_unit + scale = ( + *component_scales, + *(1.0 for _ in payload_axis_labels), + *spacing.spacing_for_ndim(2), + ) return { - "scale": (*(1.0 for _ in range(prefix)), *spacing.spacing_for_ndim(2)), + "scale": scale, + "translate": self.translate(payload_axis_labels, scale=scale), "units": ( - *("dimensionless" for _ in range(prefix)), + *component_units, + *("dimensionless" for _ in payload_axis_labels), *(spacing.native_coordinate_unit for _ in range(2)), ), } @@ -1901,7 +1925,7 @@ def append( for column_values in self.values.values(): column_values.append(None) - for name, value in metadata.items(): + for name, value in ROIArchiveSourceMetadata.feature_metadata(metadata).items(): if name in ( ObjectArtifactSubjectBinding.SUBJECT_FEATURE, ObjectArtifactSubjectBinding.SUBJECT_ID_FEATURE, diff --git a/openhcs/runtime/napari_viewer_server.py b/openhcs/runtime/napari_viewer_server.py index e16cb3e6b0..ef58616bd1 100644 --- a/openhcs/runtime/napari_viewer_server.py +++ b/openhcs/runtime/napari_viewer_server.py @@ -1164,13 +1164,14 @@ def _build_nd_points( all_properties["component"].append( metadata.value(VisualMetadataField.COMPONENT, 0) ) - point_metadata.append(metadata.metadata) + point_metadata.append( + ROIArchiveSourceMetadata.feature_metadata(metadata.metadata) + ) excluded = { "label", "component", ROIFractionalZ.FIELD, - ROIArchiveSourceMetadata.FIELD, } feature_fields = { key @@ -1840,14 +1841,12 @@ def handle(self, request: NapariLayerDisplayRequest) -> None: presentation, payload_axis_labels=payload_axis_labels, ) - translate = presentation.translate(payload_axis_labels) request = replace(request, presentation=presentation) axis_labels = presentation.axis_labels layer_kwargs = image_presentation.layer_kwargs(request.display_config.colormap) if axis_labels is not None: layer_kwargs["axis_labels"] = axis_labels - layer_kwargs["translate"] = translate layer_kwargs.update( presentation.spatial_layer_kwargs(layer_items, payload_axis_labels) ) @@ -2026,7 +2025,6 @@ def display_work( "face_color_cycle": color_projection.cycle, "opacity": 0.7, "ndim": shape_payload.ndim, - "translate": presentation.translate(), "visible": False, } if shape_payload.result_metadata: @@ -2062,7 +2060,7 @@ def geometric_component_values( items: Sequence[NapariStreamLayerItem], component_axis_semantics: ViewerComponentAxisSemantics, ) -> ComponentValues: - """Keep the declared Z span containing fractional-Z points navigable.""" + """Keep the represented source span, not just occupied point Z, navigable.""" z_component = AllComponents.Z_INDEX.value coordinates = tuple( (item, fractional_z) @@ -2083,7 +2081,7 @@ def geometric_component_values( domain = component_axis_semantics.required_component_values((z_component,))[ z_component ] - occupied: set[ComponentValue] = set() + represented: set[ComponentValue] = set() for item, fractional_z in coordinates: anchor = ViewerComponentCoordinateAuthority.required_value( item.address.components, @@ -2103,8 +2101,8 @@ def geometric_component_values( f"Fractional-Z point {fractional_z.value!r} exceeds the " f"declared Z domain from anchor {anchor!r}." ) - occupied.update(domain[start : stop + 1]) - return {z_component: [value for value in domain if value in occupied]} + represented.update(domain[start:]) + return {z_component: [value for value in domain if value in represented]} def handle(self, request: NapariLayerDisplayRequest) -> None: pipeline = request.pipeline @@ -2126,7 +2124,6 @@ def handle(self, request: NapariLayerDisplayRequest) -> None: "properties": properties, "face_color": "green", "size": 3, - "translate": presentation.translate(), } layer_kwargs.update(presentation.spatial_layer_kwargs(request.items)) if axis_labels is not None: @@ -2267,7 +2264,9 @@ def reconcile_mounted_axis_projections( if not apply: continue layer = self.server.layer_route_state.layer(route_key) - layer.translate = presentation.translate(presentation.payload_axis_labels) + layer.translate = presentation.spatial_layer_kwargs( + items, presentation.payload_axis_labels + )["translate"] self.dimension_label_store.apply(presentation) def schedule_layer_update( diff --git a/tests/unit/test_live_fractional_z_point_source.py b/tests/unit/test_live_fractional_z_point_source.py index 51b8db4723..eb2dfca0d7 100644 --- a/tests/unit/test_live_fractional_z_point_source.py +++ b/tests/unit/test_live_fractional_z_point_source.py @@ -166,7 +166,7 @@ def test_original_point_writer_live_domain_anchor_and_reopen(tmp_path, planes, o ) handler = NapariPointsLayerDisplayHandler() occupied = handler.geometric_component_values([item], authority.component_axis_semantics) - assert occupied == {'z_index': list(range(origin, origin + (3 if planes == 4 else 1)))} + assert occupied == {'z_index': list(range(origin, origin + planes))} with pytest.raises(ValueError, match='exceeds the declared Z domain'): handler.geometric_component_values( [replace(item, data=NapariROIConverter.rois_to_shapes([ diff --git a/tests/unit/test_napari_streaming_handlers.py b/tests/unit/test_napari_streaming_handlers.py index bb9d296666..4e3a241559 100644 --- a/tests/unit/test_napari_streaming_handlers.py +++ b/tests/unit/test_napari_streaming_handlers.py @@ -6251,6 +6251,92 @@ def test_napari_points_layer_uses_exact_fractional_z_from_native_roi_metadata(): assert tuple(server.viewer.calls[-1][1][0]) == (2.375, 1.25, 3.5) +@pytest.mark.parametrize( + "values,unit,expected_scale,expected_units", + ( + ((2.0, 0.65, 0.65), "micrometers", (1.0, 2.0, 0.65, 0.65), + ("dimensionless", "micrometer", "micrometer", "micrometer")), + ((2.0, 1.0, 1.0), "relative", (1.0, 2.0, 1.0, 1.0), + ("dimensionless", "dimensionless", "dimensionless", "dimensionless")), + ((0.65, 0.65), "micrometers", (1.0, 1.0, 0.65, 0.65), + ("dimensionless", "dimensionless", "micrometer", "micrometer")), + ((), "micrometers", (1.0, 1.0, 1.0, 1.0), + ("dimensionless", "dimensionless", "pixel", "pixel")), + ), +) +def test_spatial_presentation_owns_scale_and_world_translation( + values, unit, expected_scale, expected_units +): + from openhcs.core.source_metadata import SourceVoxelSpacing, SourceVoxelSpacingUnit + + presentation = _axis_presentation( + layer_key="own-calibrated-source", + projected_axis_components=("channel", "z_index"), + component_values={"channel": [4], "z_index": [2, 3, 4, 5]}, + axis_offsets=(3, 2), + ) + item = _layer_item( + {"channel": 4, "z_index": 2}, np.zeros((5, 7), dtype=np.uint16) + ) + item = replace(item, image_metadata=ImagePayloadMetadata( + source_voxel_spacing=SourceVoxelSpacing(values, SourceVoxelSpacingUnit(unit)) + )) + kwargs = presentation.spatial_layer_kwargs([item]) + assert kwargs["scale"] == expected_scale + assert kwargs["units"] == expected_units + assert kwargs["translate"] == (3.0, 2 * expected_scale[1], 0.0, 0.0) + payload_kwargs = presentation.spatial_layer_kwargs([item], ("band",)) + assert payload_kwargs["scale"] == (*expected_scale[:2], 1.0, *expected_scale[2:]) + assert payload_kwargs["units"] == (*expected_units[:2], "dimensionless", *expected_units[2:]) + + +def test_calibrated_native_image_point_and_semantic_navigation_agree_without_qt(): + from napari.layers import Image, Points + from openhcs.core.source_metadata import SourceVoxelSpacing + + presentation = _axis_presentation( + layer_key="own-calibrated-source", + projected_axis_components=("z_index",), + component_values={"z_index": [0, 1, 2, 3]}, + ) + item = replace( + _layer_item({"z_index": 0}, np.zeros((5, 7), dtype=np.uint16)), + image_metadata=ImagePayloadMetadata( + source_voxel_spacing=SourceVoxelSpacing((2.0, 0.65, 0.65)) + ), + ) + kwargs = presentation.spatial_layer_kwargs([item]) + pixels = np.zeros((4, 5, 7), dtype=np.uint16) + point = np.array([[1.5, 1.5, 2.5]]) + raw = Image(pixels, **kwargs) + centres = Points(point, **kwargs) + assert raw.data is pixels + np.testing.assert_array_equal(centres.data, point) + assert raw.data_to_world((1.5, 1.5, 2.5)) == centres.data_to_world(point[0]) + np.testing.assert_allclose(centres.data_to_world(point[0]), (3.0, 0.975, 1.625)) + dims = Dims(ndim=3, range=((0, 6, 2), (0, 2.6, 0.65), (0, 3.9, 0.65))) + dims.current_step = (presentation.viewer_step(2, 0), 0, 0) + assert dims.point[0] == 4.0 + assert raw.world_to_data(dims.point)[0] == 2.0 + state = NapariDimensionLayerState( + labels={"z_index": ["Z0", "Z1", "Z2", "Z3"]}, + presentation=presentation, + ) + assert state.label_parts_for_current_step(dims.current_step) == ("Z2",) + + +def test_shape_features_exclude_source_transport_without_mutating_payload(): + from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata + + full_metadata = {"label": 7, "response": 0.25, + ROIArchiveSourceMetadata.FIELD: {"source_provenance": "retained"}} + features = NapariShapeFeatureColumns() + features.append(full_metadata, label=7, path="/own/synthetic.roi.zip") + assert ROIArchiveSourceMetadata.FIELD in full_metadata + assert ROIArchiveSourceMetadata.FIELD not in features.values + assert features.values["response"] == [0.25] + + def test_napari_fractional_z_points_are_relative_to_the_declared_anchor(): napari_viewer_server = pytest.importorskip("openhcs.runtime.napari_viewer_server") server = _FakeNapariServer() diff --git a/tests/unit/test_streaming_service.py b/tests/unit/test_streaming_service.py index 9625eb1834..1b25360f8b 100644 --- a/tests/unit/test_streaming_service.py +++ b/tests/unit/test_streaming_service.py @@ -954,9 +954,10 @@ def test_reopen_native_roi_archives_preserves_per_file_source_and_calibration( stream.source.metadata.metadata_by_path[batch_paths[0]] == expected.source_component_metadata ) - assert data[0][0].metadata == { + assert ROIArchiveSourceMetadata.feature_metadata(data[0][0].metadata) == { "label": expected.source_component_metadata["channel"] } + assert ROIArchiveSourceMetadata.decode(data[0]) == expected assert data[0][0].shapes == [PointShape(32.25, 40.5)] @@ -1047,6 +1048,9 @@ def test_3d_point_archive_reopens_with_native_z_domain_and_features(tmp_path): assert paths == [archive] assert data[0][0].metadata["response"] == 4.75 assert data[0][0].metadata["openhcs_fractional_z"] == 2.375 + reopened_source = ROIArchiveSourceMetadata.decode(data[0]) + assert reopened_source is not None + assert reopened_source.source_provenance == table.source_provenance assert stream.source.metadata.metadata_by_path[archive]["z_index"] == 0 assert stream.message_extra[ViewerBatchWireField.COMPONENT_VALUE_DOMAIN.value][ "z_index" From c7a11af98feec16185e176d14340bc5dbf5d6f00 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 03:20:04 -0400 Subject: [PATCH 124/125] test(viewer): compare reopen with original writer archive identity --- tests/unit/test_streaming_service.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_streaming_service.py b/tests/unit/test_streaming_service.py index 1b25360f8b..5f801971c6 100644 --- a/tests/unit/test_streaming_service.py +++ b/tests/unit/test_streaming_service.py @@ -9,7 +9,7 @@ import pytest from polystore.disk import DiskStorageBackend from polystore.filemanager import FileManager -from polystore.roi import ROI, PointShape +from polystore.roi import ROI, PointShape, load_rois_from_zip from polystore.roi_converters import NapariROIConverter from polystore.streaming.identity import ( FixedStreamProducerIdentityKind, @@ -1031,6 +1031,7 @@ def test_3d_point_archive_reopens_with_native_z_domain_and_features(tmp_path): ), ) config = NapariStreamingConfig(enabled=True) + saved_source = ROIArchiveSourceMetadata.decode(load_rois_from_zip(Path(archive))) result = StreamingService(filemanager, handler, tmp_path).stream_rois( RoiStreamingRequest( viewer=FakeViewer(), @@ -1050,7 +1051,11 @@ def test_3d_point_archive_reopens_with_native_z_domain_and_features(tmp_path): assert data[0][0].metadata["openhcs_fractional_z"] == 2.375 reopened_source = ROIArchiveSourceMetadata.decode(data[0]) assert reopened_source is not None - assert reopened_source.source_provenance == table.source_provenance + assert reopened_source == saved_source + assert ( + reopened_source.source_image_provenance_planes + == table.source_image_provenance_planes + ) assert stream.source.metadata.metadata_by_path[archive]["z_index"] == 0 assert stream.message_extra[ViewerBatchWireField.COMPONENT_VALUE_DOMAIN.value][ "z_index" From e6c0bf77f8e4d24d80c0463e214070bc018a3301 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 3 Oct 2026 03:23:28 -0400 Subject: [PATCH 125/125] refactor(viewer): require world scale at presentation translation owner --- openhcs/runtime/napari_streaming_handlers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openhcs/runtime/napari_streaming_handlers.py b/openhcs/runtime/napari_streaming_handlers.py index 2c7b7e7338..4047ed9ff3 100644 --- a/openhcs/runtime/napari_streaming_handlers.py +++ b/openhcs/runtime/napari_streaming_handlers.py @@ -1442,7 +1442,7 @@ def translate( self, payload_axis_labels: tuple[str, ...] = (), *, - scale: Sequence[float] | None = None, + scale: Sequence[float], ) -> tuple[float, ...]: """Place route offsets in the same native world units as layer scale.""" @@ -1455,8 +1455,6 @@ def translate( 0.0, 0.0, ) - if scale is None: - return offsets return tuple( offset * spacing for offset, spacing in zip(offsets, scale, strict=True) )