From fb506302eb6aa90cb8175ce33d99e3f457819622 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 03:07:02 -0400 Subject: [PATCH 1/3] Document native result reopening ownership and acceptance --- .../materialized_result_reopening_20260929.md | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 paper/plans/materialized_result_reopening_20260929.md diff --git a/paper/plans/materialized_result_reopening_20260929.md b/paper/plans/materialized_result_reopening_20260929.md new file mode 100644 index 0000000000..34b22f4b3e --- /dev/null +++ b/paper/plans/materialized_result_reopening_20260929.md @@ -0,0 +1,57 @@ +# Native materialized-result reopening + +Owner: OpenHCS blind-analysis integration coordinator. Issue #134. +Base: 9a6db7bab414740f91d953367ab64f0cc6106026, merged inspection PR #173. +Branch: fix/mcp-materialized-result-reopening-20260929. +Worktree: /home/ts/wt/openhcs-materialized-result-route-20260929. + +Required workflow remains the full workflow in materialized_result_followup_20260929.md. +This continuation must not narrow it to file inspection or a synthetic count. + +## Ownership decision + +Task authority: reopen exact retained materialized files without a scientific +rerun, relocation, filename-derived source identity or coordinate/calibration loss. +The initial NRA class-first corpus covers the plate DTO, inspection service, +streaming service and core viewer streaming service: 75 original ClassDefs, +75 unique canonical class-family joins, no omitted syntax rows. Global R1/MRO +and native equivalence remain OPEN. Feature extensions are intentional authored +changes, not a proved-equivalent NRA transaction. + +Determining owners and required consumers: + +- AgentPathPolicy -> the one admitted explicit-directory inventory used by both + query and streaming. Do not duplicate the reader/record-building algorithm. +- PlateFileStreamRequest -> automatically generated MCP selection fields. +- Original PlateInspectionContext -> StreamingService source calibration. The + independent result-directory selection must not replace this source context. +- ImagePayloadMetadata -> native ROI source/plane/domain/spacing projection. + Its existing codec determines meanings; a new archive codec may encode/decode + this owner, but may not recreate its schema, registry or field roster. +- Existing PolyStore ROI codec -> geometry and object identities. No fallback + conversion of malformed archives and no invented detections. + +IDEN-1: plate/source context and persisted output location are different facts. +BOUND-2: source metadata must descend to ImagePayloadMetadata, not raw consumers. +Counterevidence: ROI geometry is preserved by the native archive, but current +Output.metadata does not enter the per-ROI disk sidecar. A reader cannot recover +discarded plane/source identity by making plausible filename guesses. + +Historical external ROI files can lack OpenHCS source metadata. That is a genuine +optional external contract, not a compatibility reader. Exact arbitrary-directory +reopening must reject missing required source/plane evidence rather than claim it. +H002's historical malformed centre archive remains unchanged and unaccepted. + +## Acceptance and crossing + +Run focused source/native disk round-trip checks, then the installed public MCP +journey on bounded native producer outputs: explicit directory, original source, +exact labels/coordinates, plane identities, transforms, and personally opened +raw/result/combined captures under two windows. Check frozen science digests. +Use isolated :90/unused port, not the active H002 port 5690 or desktop :0. +Keep the held-out and private-reference boundaries intact. + +PR #157 still owns worker_execution, runtime_exports and +function_artifact_materialization. Do not compete there. This branch owns the +independent shared directory consumer and native source-metadata persistence +boundary. Full writer-success observation remains coordinated with #157. From 9f37cb66d6ed2255e06a4d497f5e560c2fe01817 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 03:23:24 -0400 Subject: [PATCH 2/3] Reopen native ROI results with persisted source metadata --- openhcs/agent/capabilities.py | 5 + openhcs/agent/dto/plate.py | 4 + .../services/plate_inspection_service.py | 35 +++--- .../agent/services/plate_streaming_service.py | 44 ++++++- openhcs/core/roi_source_metadata.py | 52 ++++++++ openhcs/core/viewer_streaming_service.py | 104 +++++++++++----- openhcs/processing/materialization/core.py | 12 +- .../create_native_result_reopening_fixture.py | 101 ++++++++++++++++ tests/unit/agent/test_capabilities.py | 17 +++ .../agent/test_plate_streaming_service.py | 76 +++++++++++- tests/unit/test_roi_source_metadata.py | 100 ++++++++++++++++ tests/unit/test_streaming_service.py | 112 +++++++++++++++++- 12 files changed, 602 insertions(+), 60 deletions(-) create mode 100644 openhcs/core/roi_source_metadata.py create mode 100644 tests/diagnostics/create_native_result_reopening_fixture.py create mode 100644 tests/unit/test_roi_source_metadata.py diff --git a/openhcs/agent/capabilities.py b/openhcs/agent/capabilities.py index 1abc22dc18..e1fc286a4c 100644 --- a/openhcs/agent/capabilities.py +++ b/openhcs/agent/capabilities.py @@ -2132,8 +2132,13 @@ class StreamPlateFilesToViewerCapability(PlatePathCapability): "Resolves image or ROI result records by virtual path, source path, " "result path, basename, or bounded inventory query, then streams them " "to a managed viewer through the same core service used by the Image Browser." + " Set result_directory to reopen retained native results independently of " + "their output location; plate_path supplies the original source context. " + "ROI reopening requires persisted source metadata, not filename guesses." ) service = "plate_streaming" + mutating = True + side_effects = ("launches_or_updates_managed_viewer",) data_exposure = ( "local_plate_path", "plate_virtual_image_path", diff --git a/openhcs/agent/dto/plate.py b/openhcs/agent/dto/plate.py index a600d19719..50beb38243 100644 --- a/openhcs/agent/dto/plate.py +++ b/openhcs/agent/dto/plate.py @@ -383,6 +383,7 @@ class PlateFileStreamRequest: """Stream image or ROI files exposed by a local plate inventory to a viewer.""" plate_path: str + result_directory: str | None = None context_plate_path: str | None = None file_paths: tuple[str, ...] = () microscope_type: str = PlateInspectionDefaults.MICROSCOPE_AUTO @@ -401,6 +402,7 @@ def from_fields( cls, *, plate_path: str, + result_directory: str | None = None, file_paths: list[str] | None = None, microscope_type: str = PlateInspectionDefaults.MICROSCOPE_AUTO, pattern_format: str | None = None, @@ -418,6 +420,7 @@ def from_fields( ) -> "PlateFileStreamRequest": return cls( plate_path=plate_path, + result_directory=result_directory, file_paths=tuple(file_paths or ()), microscope_type=microscope_type, pattern_format=pattern_format, @@ -439,6 +442,7 @@ def from_fields( def as_tool_arguments(self) -> dict[str, JsonValue]: return { "plate_path": self.plate_path, + "result_directory": self.result_directory, "file_paths": list(self.file_paths) if self.file_paths else None, "microscope_type": self.microscope_type, "pattern_format": self.pattern_format, diff --git a/openhcs/agent/services/plate_inspection_service.py b/openhcs/agent/services/plate_inspection_service.py index 6cdf7b1c3e..dae396298f 100644 --- a/openhcs/agent/services/plate_inspection_service.py +++ b/openhcs/agent/services/plate_inspection_service.py @@ -1258,17 +1258,8 @@ def _query_explicit_result_directory( ) if result_path is None: raise RuntimeError("Result directory resolution returned no path or error.") - from openhcs.microscopes.microscope_interfaces import AnalysisResultDirectory - try: - result_inventory = PlateResultFileInventory.from_directory_files( - plate_path=result_path, - result_directory=AnalysisResultDirectory( - subdirectory_name=result_path.name, - path=result_path, - ), - file_paths=self._path_policy.iter_readable_files(result_path), - ) + file_inventory = self.result_directory_inventory(result_path) except AgentPathPolicyError as exc: return self._query_files_error( request, @@ -1282,16 +1273,32 @@ def _query_explicit_result_directory( return self._query_files_from_inventory( request=request, plate_path=plate_path, - file_inventory=PlateFileInventory.from_inventories( - image_inventory=PlateImageInventory(plate_path=result_path, records=()), - result_inventory=result_inventory, - ), + file_inventory=file_inventory, detected_microscope_type=None, handler_class=None, parser_class=None, warnings=(), ) + def result_directory_inventory(self, directory: Path) -> PlateFileInventory: + """Admit persisted files once for both inspection and viewer reopening.""" + from openhcs.microscopes.microscope_interfaces import AnalysisResultDirectory + + result_path = self._path_policy.assert_readable(directory) + if not result_path.is_dir(): + raise ValueError(f"Result path is not a directory: {result_path}") + return PlateFileInventory.from_inventories( + image_inventory=PlateImageInventory(plate_path=result_path, records=()), + result_inventory=PlateResultFileInventory.from_directory_files( + plate_path=result_path, + result_directory=AnalysisResultDirectory( + subdirectory_name=result_path.name, + path=result_path, + ), + file_paths=self._path_policy.iter_readable_files(result_path), + ), + ) + @staticmethod def _query_files_from_inventory( *, diff --git a/openhcs/agent/services/plate_streaming_service.py b/openhcs/agent/services/plate_streaming_service.py index 1e9580e8c9..0f7696cacf 100644 --- a/openhcs/agent/services/plate_streaming_service.py +++ b/openhcs/agent/services/plate_streaming_service.py @@ -38,6 +38,7 @@ from openhcs.constants import AllComponents, Backend from openhcs.core.config import StreamingConfig from openhcs.core.plate_image_inventory import ( + PlateFileKind, PlateFileInventoryQuery, PlateFileRecord, ) @@ -143,12 +144,26 @@ def stream_files( transport_mode=config.transport_mode, persistent=config.persistent, ) - inventory, inventory_warnings = ( - self._plate_inspection_service.file_inventory( - stream_context, - kind=None if request.file_paths else request.kind, + if request.result_directory is not None: + if request.kind is not PlateFileKind.RESULT or request.well is not None: + raise ValueError( + "Explicit result-directory streaming requires kind='result' " + "and no acquisition-component filter." + ) + result_path = self._plate_inspection_service.resolve_readable_path( + request.result_directory + ) + inventory = self._plate_inspection_service.result_directory_inventory( + result_path + ) + inventory_warnings = context.warnings + else: + inventory, inventory_warnings = ( + self._plate_inspection_service.file_inventory( + stream_context, + kind=None if request.file_paths else request.kind, + ) ) - ) resolved_records = self._resolve_records(request, inventory) ( image_paths, @@ -156,6 +171,15 @@ def stream_files( roi_component_metadata_by_path, skipped_records, ) = self._streamable_paths(resolved_records) + if ( + request.result_directory is not None + and image_paths + and request.source_receipt is None + ): + raise ValueError( + "Explicit result images require an exact source receipt; " + "artifact filenames do not establish source identity." + ) all_warnings = inventory_warnings if skipped_records: all_warnings = ( @@ -201,6 +225,8 @@ def stream_files( read_backend = stream_context.handler.get_primary_backend( stream_context.plate_path, stream_context.filemanager ) + if request.result_directory is not None: + read_backend = Backend.DISK.value source_projection, producer = self._receipt_source_projection( request, resolved_records, stream_context ) @@ -240,7 +266,12 @@ def stream_files( for record in resolved_records if record.streamable_roi_path is not None ) - streaming_service.stream_rois( + roi_streaming_service = StreamingService( + filemanager=context.filemanager, + microscope_handler=context.handler, + plate_path=context.plate_path, + ) + roi_streaming_service.stream_rois( RoiStreamingRequest( viewer=viewer, config=config, @@ -249,6 +280,7 @@ def stream_files( roi_filenames=roi_paths, component_metadata_by_path=roi_component_metadata_by_path, producer=roi_producer, + require_source_metadata=request.result_directory is not None, ) ) except Exception as exc: diff --git a/openhcs/core/roi_source_metadata.py b/openhcs/core/roi_source_metadata.py new file mode 100644 index 0000000000..6455aaadf5 --- /dev/null +++ b/openhcs/core/roi_source_metadata.py @@ -0,0 +1,52 @@ +"""Persist the existing image-source declaration in native ROI metadata.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import replace +from typing import ClassVar + +from polystore.roi import ROI + +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.serialization.json import to_jsonable + + +class ROIArchiveSourceMetadata: + """Native ROI sidecar projection, not a second source-metadata schema.""" + + FIELD: ClassVar[str] = "openhcs_image_payload_metadata" + + @classmethod + def bind(cls, rois: Sequence[ROI], metadata: ImagePayloadMetadata) -> list[ROI]: + encoded = to_jsonable(metadata) + return [ + replace(roi, metadata={**roi.metadata, cls.FIELD: encoded}) for roi in rois + ] + + @classmethod + def decode(cls, rois: Sequence[ROI]) -> ImagePayloadMetadata | None: + """Decode once; external ROI archives may have no OpenHCS source binding.""" + encoded = [roi.metadata.get(cls.FIELD) for roi in rois] + if not any(value is not None for value in encoded): + return None + if any(value is None or value != encoded[0] for value in encoded): + raise ValueError( + "ROI archive contains missing or conflicting source metadata." + ) + return ImagePayloadMetadata.from_mapping(encoded[0]) + + @classmethod + def geometry(cls, rois: Sequence[ROI]) -> list[ROI]: + """Keep transport-only source declarations out of the ROI feature table.""" + return [ + replace( + roi, + metadata={ + key: value + for key, value in roi.metadata.items() + if key != cls.FIELD + }, + ) + for roi in rois + ] diff --git a/openhcs/core/viewer_streaming_service.py b/openhcs/core/viewer_streaming_service.py index 9339a0cca3..06c7d64c57 100644 --- a/openhcs/core/viewer_streaming_service.py +++ b/openhcs/core/viewer_streaming_service.py @@ -30,6 +30,7 @@ from zmqruntime.viewer_protocol import ViewerWireMapping from openhcs.core.runtime_image_loading import ImagePayloadSourceMetadataContext +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, image_payload_data, @@ -129,6 +130,7 @@ class RoiStreamingRequest(ViewerStreamingContext): default_factory=dict ) producer: ViewerStreamProducer | None = None + require_source_metadata: bool = False @dataclass(frozen=True, slots=True) @@ -331,6 +333,16 @@ def plate_image_metadata(self) -> ImagePayloadMetadata: source_voxel_spacing=SourceVoxelSpacing((pixel_size, pixel_size)) ) + def calibrated_metadata( + self, metadata: ImagePayloadMetadata | None + ) -> ImagePayloadMetadata: + """Fill absent spacing from the acquisition without replacing native facts.""" + if metadata is None: + return self.plate_image_metadata() + if metadata.source_voxel_spacing.has_values: + return metadata + return metadata.with_source_spatial_context_from(self.plate_image_metadata()) + def load_image( self, filename: str, @@ -358,10 +370,7 @@ def load_image( filemanager=self.filemanager, source_address=source_path, ).metadata(image) - if not metadata.source_voxel_spacing.has_values: - metadata = metadata.with_source_spatial_context_from( - self.plate_image_metadata() - ) + metadata = self.calibrated_metadata(metadata) return metadata.payload_with( image_payload_data(image), image_payload_mask(image) ) @@ -768,6 +777,7 @@ def stream_rois( data_list: list = [] paths: list[str] = [] loaded_indices: list[int] = [] + archive_metadata: list[ImagePayloadMetadata | None] = [] for i, filename in enumerate(request.roi_filenames, 1): file_path = Path(self.source.plate_path) / filename @@ -776,7 +786,17 @@ def stream_rois( logger.warning(f"No ROIs found in {file_path.name}") continue - data_list.append(rois) + metadata = ROIArchiveSourceMetadata.decode(rois) + if request.require_source_metadata and ( + metadata is None + or not metadata.source_provenance.represented_source_identities + ): + raise ValueError( + f"Native ROI source metadata is required to reopen {file_path}; " + "artifact filenames do not establish source identity." + ) + archive_metadata.append(metadata) + data_list.append(ROIArchiveSourceMetadata.geometry(rois)) paths.append(filename) loaded_indices.append(i - 1) @@ -811,9 +831,16 @@ def stream_rois( self.source, self.transport_config, ) + external_paths = [ + path + for path, metadata in zip(paths, archive_metadata, strict=True) + if metadata is None + ] if request.component_metadata_by_path: missing_metadata = tuple( - path for path in paths if path not in request.component_metadata_by_path + path + for path in external_paths + if path not in request.component_metadata_by_path ) if missing_metadata: raise ValueError( @@ -821,10 +848,24 @@ def stream_rois( f"metadata was missing for {missing_metadata!r}." ) metadata_by_path = { - path: dict(request.component_metadata_by_path[path]) for path in paths + path: dict(request.component_metadata_by_path[path]) + for path in external_paths } else: - metadata_by_path = self.source.roi_component_metadata_by_path(paths) + metadata_by_path = self.source.roi_component_metadata_by_path( + external_paths + ) + metadata_by_path.update( + ( + path, + dict( + metadata.source_provenance.scalar_source_identity.component_metadata + or {} + ), + ) + for path, metadata in zip(paths, archive_metadata, strict=True) + if metadata is not None + ) source_metadata_items = StreamSourceComponentMetadataItems.from_values( metadata_by_path[path] for path in paths ) @@ -838,31 +879,36 @@ def stream_rois( "selected_rois", ) ) - stream_backend_kwargs = message_authority.viewer_backend_kwargs( - producer=producer.for_indices( - loaded_indices, - total, - ), - source_metadata=message_authority.path_mapped_source_metadata( - metadata_by_path - ), - ).with_item_fields( - StreamImagePayloadMetadataProjector.item_fields( - self.source.plate_image_metadata(), - message_authority.layout.component_order, - ) - ) - message = f"Streaming {len(paths)} ROI file(s) to {display_name}..." messages.append(message) request.status_callback(message) - self.source.filemanager.save_batch( - data_list, - paths, - backend_enum.value, - **stream_backend_kwargs.to_kwargs(), - ) + payload_metadata = [ + self.source.calibrated_metadata(metadata) + for metadata in archive_metadata + ] + component_order = message_authority.layout.component_order + for indices in StreamImagePayloadMetadataProjector.partition_indices( + payload_metadata, component_order + ): + item_fields = StreamImagePayloadMetadataProjector.item_fields( + payload_metadata[indices[0]], component_order + ) + stream_backend_kwargs = message_authority.viewer_backend_kwargs( + producer=producer.for_indices( + tuple(loaded_indices[index] for index in indices), total + ), + source_metadata=message_authority.path_mapped_source_metadata( + {paths[index]: metadata_by_path[paths[index]] for index in indices}, + item_fields=item_fields, + ), + ).with_item_fields(item_fields) + self.source.filemanager.save_batch( + [data_list[index] for index in indices], + [paths[index] for index in indices], + backend_enum.value, + **stream_backend_kwargs.to_kwargs(), + ) self._require_viewer_settled(request) message = ( diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index 437f42ab19..137a7dd9c4 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -2942,6 +2942,7 @@ def _write_roi_zip( ctx: MaterializationContext, ) -> list[Output]: from polystore.roi import extract_rois_from_labeled_mask + from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata request = ROIMaterializationTargetRequest.from_context(data, options, ctx) materialization_input = request.materialization_input @@ -3017,15 +3018,14 @@ def _write_roi_zip( source_component_metadata=source_identity.component_metadata, ).with_missing_from(item_metadata.source_provenance) ) + item_metadata = item_metadata.replace_fields( + source_spatial_domain=source_domain_authority.domain_for_target(target) + ) outs.append( Output( path=target.archive.path, - content=target_rois, - metadata=item_metadata.replace_fields( - source_spatial_domain=( - source_domain_authority.domain_for_target(target) - ), - ), + content=ROIArchiveSourceMetadata.bind(target_rois, item_metadata), + metadata=item_metadata, ) ) diff --git a/tests/diagnostics/create_native_result_reopening_fixture.py b/tests/diagnostics/create_native_result_reopening_fixture.py new file mode 100644 index 0000000000..1909929737 --- /dev/null +++ b/tests/diagnostics/create_native_result_reopening_fixture.py @@ -0,0 +1,101 @@ +"""Create native transport fixtures, never detections, beside a synthetic plate.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import ROI, PointShape, load_rois_from_zip + +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_object_label_building import ( + SourceImageObjectLabelBuildRequest, +) +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.processing.materialization import ( + MaterializationSpec, + ROIOptions, + materialize, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("plate_path", type=Path) + args = parser.parse_args() + source = args.plate_path / "TimePoint_1/A01_s001_w1_z001_t001.tif" + if not source.is_file(): + parser.error(f"Synthetic raw image missing: {source}") + target = args.plate_path / "retained_native_20260929" + if target.exists(): + parser.error(f"Refusing to overwrite an existing fixture: {target}") + metadata = ImagePayloadMetadata( + source_path=str(source), + source_component_metadata={ + "well": "A01", + "site": 1, + "channel": 1, + "z_index": 1, + "timepoint": 1, + }, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(0, 0), source_shape_yx=(128, 128) + ), + source_voxel_spacing=SourceVoxelSpacing((0.65, 0.65)), + ) + coordinates = ((32.25, 40.5), (64.5, 80.25), (96.75, 64.0)) + points = [ + ROI( + [PointShape(y, x)], + { + "label": label, + "fixture_purpose": "synthetic transport only; not cell detections", + }, + ) + for label, (y, x) in enumerate(coordinates, 1) + ] + disk = DiskStorageBackend() + archive = target / "misleading_B99_w7_points.roi.zip" + disk.save(ROIArchiveSourceMetadata.bind(points, metadata), archive) + assert ( + ROIArchiveSourceMetadata.decode(load_rois_from_zip(archive)).source_provenance + == metadata.source_provenance + ) + labels = np.zeros((128, 128), dtype=np.int32) + for label, (y, x) in enumerate(coordinates, 1): + labels[int(y) - 2 : int(y) + 3, int(x) - 2 : int(x) + 3] = label + payload = SourceImageObjectLabelBuildRequest( + image=metadata.payload_with(np.zeros_like(labels)), labels=labels + ).payload() + materialized = materialize( + MaterializationSpec(ROIOptions(min_area=0)), + data=payload, + path=str(target / "misleading_B99_w7_labels.roi.zip"), + filemanager=FileManager({"disk": disk}), + backends=["disk"], + backend_kwargs={}, + ) + restored = ROIArchiveSourceMetadata.decode(load_rois_from_zip(Path(materialized))) + assert restored.source_provenance == metadata.source_provenance + assert restored.source_voxel_spacing == metadata.source_voxel_spacing + print( + json.dumps( + { + "result_directory": str(target), + "points_archive": str(archive), + "materialized_archive": materialized, + "coordinates_yx": coordinates, + "purpose": "native transport only; no scientific detector or acceptance", + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/agent/test_capabilities.py b/tests/unit/agent/test_capabilities.py index 47995a3578..08163b733f 100644 --- a/tests/unit/agent/test_capabilities.py +++ b/tests/unit/agent/test_capabilities.py @@ -40,6 +40,23 @@ def test_capability_registry_declares_schema_and_unique_names(): assert len(names) == len(set(names)) +def test_manual_streaming_declares_viewer_side_effects_and_directory_request(): + from openhcs.agent.capabilities import StreamPlateFilesToViewerCapability + from openhcs.agent.dto.plate import PlateFileStreamRequest + from inspect import signature + + capability = next( + cap + for cap in get_capability_registry().capabilities + if cap.name == StreamPlateFilesToViewerCapability.name + ) + assert not capability.read_only + assert capability.side_effects == ("launches_or_updates_managed_viewer",) + assert ( + "result_directory" in signature(PlateFileStreamRequest.from_fields).parameters + ) + + def test_source_session_capability_owns_progress_heartbeat_policy(): capabilities = { capability.name: capability diff --git a/tests/unit/agent/test_plate_streaming_service.py b/tests/unit/agent/test_plate_streaming_service.py index 783b76a8ee..256f231297 100644 --- a/tests/unit/agent/test_plate_streaming_service.py +++ b/tests/unit/agent/test_plate_streaming_service.py @@ -62,6 +62,14 @@ def file_inventory(self, context, *, kind): self.inventory_kinds.append(kind) return self.inventory, () + def resolve_readable_path(self, path): + self.resolve_requests.append(path) + return Path(path) + + def result_directory_inventory(self, path): + assert path == self.inventory.plate_path + return self.inventory + class FakeViewer: port = 5565 @@ -617,8 +625,74 @@ def fake_stream_rois(self, request): assert inspection_service.inventory_contexts[0].plate_path == Path("/plate_openhcs") assert inspection_service.inventory_kinds == [None] assert captured == { - "stream_plate_path": Path("/plate_openhcs"), + "stream_plate_path": Path("/plate"), "roi_filenames": (roi_full_path,), "component_metadata_by_path": {}, } assert result.status_messages == ("streamed output rois",) + + +def test_explicit_result_route_keeps_source_context_and_requires_native_binding( + monkeypatch, +): + path = "/retained/misleading_A01_w2.roi.zip" + inventory = PlateFileInventory( + plate_path=Path("/retained"), + image_records=(), + result_records=( + PlateResultFileRecord( + relative_path="misleading_A01_w2.roi.zip", + full_path=path, + file_format=FileFormat.ROI, + metadata={}, + ), + ), + ) + inspection = FakeInspectionService(inventory) + captured = {} + monkeypatch.setattr( + "openhcs.agent.services.plate_streaming_service.StreamingViewerLifecycle.get_or_create_visualizer", + lambda **_kwargs: FakeViewer(), + ) + + def capture(self, request): + captured["source"] = self.source.plate_path + captured["request"] = request + + monkeypatch.setattr( + "openhcs.agent.services.plate_streaming_service.StreamingService.stream_rois", + capture, + ) + request = PlateFileStreamRequest.from_fields( + plate_path="/actual_source", + result_directory="/retained", + kind="result", + file_paths=["misleading_A01_w2.roi.zip"], + ) + assert request.as_tool_arguments()["result_directory"] == "/retained" + result = plate_streaming_service(inspection).stream_files(request) + assert result.errors == () + assert inspection.inventory_contexts == [] + assert captured["source"] == Path("/actual_source") + assert captured["request"].require_source_metadata is True + assert captured["request"].roi_filenames == (path,) + + +def test_explicit_result_route_rejects_acquisition_selection_before_launch(monkeypatch): + import pytest + + inspection = FakeInspectionService(PlateFileInventory(Path("/retained"), (), ())) + monkeypatch.setattr( + "openhcs.agent.services.plate_streaming_service.StreamingViewerLifecycle.get_or_create_visualizer", + lambda **_kwargs: pytest.fail("Invalid selection must not launch a viewer"), + ) + result = plate_streaming_service(inspection).stream_files( + PlateFileStreamRequest.from_fields( + plate_path="/actual_source", + result_directory="/retained", + kind="result", + well="A01", + ) + ) + assert result.errors + assert "no acquisition-component filter" in result.errors[0].message diff --git a/tests/unit/test_roi_source_metadata.py b/tests/unit/test_roi_source_metadata.py new file mode 100644 index 0000000000..2b1fff511c --- /dev/null +++ b/tests/unit/test_roi_source_metadata.py @@ -0,0 +1,100 @@ +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import ROI, PointShape, load_rois_from_zip + +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_object_labels import ( + ObjectLabelPayload, + ObjectLabelVariantData, +) +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.processing.materialization import ( + MaterializationSpec, + ROIOptions, + materialize, +) + + +@pytest.mark.parametrize("spacing", [(0.65, 0.65), (2.0, 0.65, 0.65)]) +def test_native_roi_disk_roundtrip_preserves_the_source_owner(tmp_path, spacing): + metadata = ImagePayloadMetadata( + source_path="/source/actual-image.tif", + source_component_metadata={"well": "B02", "channel": 3, "z_index": 7}, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(10, 20), source_shape_yx=(100, 200) + ), + source_voxel_spacing=SourceVoxelSpacing(spacing), + ) + rois = [ROI([PointShape(32.25, 40.5)], {"label": 12, "plane_indices": (2,)})] + bound = ROIArchiveSourceMetadata.bind(rois, metadata) + path = tmp_path / "misleading_A01_w2.roi.zip" + DiskStorageBackend().save(bound, path) + restored = load_rois_from_zip(path) + decoded = ROIArchiveSourceMetadata.decode(restored) + assert decoded.source_provenance == metadata.source_provenance + assert decoded.source_voxel_spacing == metadata.source_voxel_spacing + assert decoded.source_spatial_domain == metadata.source_spatial_domain + assert ROIArchiveSourceMetadata.geometry(restored) == rois + assert rois[0].metadata == {"label": 12, "plane_indices": (2,)} + + +def test_source_metadata_is_optional_only_for_external_shapes(): + roi = ROI([PointShape(1, 2)], {"label": 1}) + assert ROIArchiveSourceMetadata.decode([roi]) is None + bound = ROIArchiveSourceMetadata.bind( + [roi], ImagePayloadMetadata(source_path="one") + ) + with pytest.raises(ValueError, match="missing or conflicting"): + ROIArchiveSourceMetadata.decode([*bound, roi]) + conflict = ROIArchiveSourceMetadata.bind( + [roi], ImagePayloadMetadata(source_path="two") + ) + with pytest.raises(ValueError, match="missing or conflicting"): + ROIArchiveSourceMetadata.decode([*bound, *conflict]) + malformed = replace(roi, metadata={ROIArchiveSourceMetadata.FIELD: {"unknown": 1}}) + with pytest.raises((TypeError, ValueError)): + ROIArchiveSourceMetadata.decode([malformed]) + + +def test_real_roi_materialization_persists_output_source_metadata(tmp_path): + labels = np.zeros((8, 8), dtype=np.int32) + labels[2:6, 3:7] = 12 + payload = ObjectLabelPayload( + variant_data=ObjectLabelVariantData(labels=labels), + source_path="/source/actual-image.tif", + source_component_metadata={ + "well": "B02", + "site": 1, + "channel": 3, + "z_index": 7, + "timepoint": 1, + }, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(10, 20), source_shape_yx=(100, 200) + ), + parent_image_source_voxel_spacing=SourceVoxelSpacing((0.65, 0.65)), + ) + path = materialize( + MaterializationSpec(ROIOptions(min_area=0)), + data=payload, + path=str(tmp_path / "labels.roi.zip"), + filemanager=FileManager({"disk": DiskStorageBackend()}), + backends=["disk"], + backend_kwargs={}, + ) + rois = load_rois_from_zip(Path(path)) + metadata = ROIArchiveSourceMetadata.decode(rois) + assert metadata.source_path == payload.source_path + assert metadata.source_component_metadata == payload.source_component_metadata + assert metadata.source_voxel_spacing == payload.parent_image_source_voxel_spacing + assert metadata.source_spatial_domain.origin_yx == (10, 20) + assert metadata.source_spatial_domain.source_shape_yx == (100, 200) + assert rois[0].metadata["label"] == 12 + assert rois[0].metadata["centroid"] == (13.5, 24.5) diff --git a/tests/unit/test_streaming_service.py b/tests/unit/test_streaming_service.py index 2567adf907..60a63b1a9b 100644 --- a/tests/unit/test_streaming_service.py +++ b/tests/unit/test_streaming_service.py @@ -7,6 +7,7 @@ import numpy as np import pytest +from polystore.roi import ROI, PointShape from polystore.streaming.identity import ( FixedStreamProducerIdentityKind, StreamProducerIdentity, @@ -34,6 +35,7 @@ ImagePayloadMetadata, image_payload_data, ) +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.core.source_metadata import SourceVoxelSpacing, SourceVoxelSpacingUnit @@ -740,7 +742,7 @@ def test_stream_rois_supplies_per_path_component_metadata_from_artifact_name( ) monkeypatch.setattr( "polystore.roi.load_rois_from_zip", - lambda _path: [object()], + lambda _path: [ROI(shapes=[PointShape(1, 2)])], ) filemanager = FakeFileManager() config = FijiStreamingConfig(enabled=True) @@ -795,7 +797,7 @@ def test_stream_rois_supplies_per_path_component_metadata_from_artifact_name( def test_stream_rois_uses_explicit_component_metadata(monkeypatch) -> None: monkeypatch.setattr( "polystore.roi.load_rois_from_zip", - lambda _path: [object()], + lambda _path: [ROI(shapes=[PointShape(1, 2)])], ) filemanager = FakeFileManager() config = FijiStreamingConfig(enabled=True) @@ -850,10 +852,108 @@ def test_stream_rois_uses_explicit_component_metadata(monkeypatch) -> None: assert image_metadata.source_voxel_spacing == SourceVoxelSpacing((1.3556, 1.3556)) +@pytest.mark.parametrize("config_type", [FijiStreamingConfig, NapariStreamingConfig]) +def test_reopen_native_roi_archives_preserves_per_file_source_and_calibration( + tmp_path, config_type, monkeypatch +): + from polystore.disk import DiskStorageBackend + + paths = [] + metadata_items = [] + for channel, spacing in [(3, (0.65, 0.65)), (4, (2.0, 0.8, 0.8))]: + metadata = ImagePayloadMetadata( + source_path=f"/actual/source/channel-{channel}.tif", + source_component_metadata={ + "well": "B02", + "site": 1, + "channel": channel, + "z_index": 7, + "timepoint": 1, + }, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(10, 20), source_shape_yx=(100, 200) + ), + source_voxel_spacing=SourceVoxelSpacing(spacing), + ) + path = tmp_path / f"misleading_A01_w{channel}.roi.zip" + DiskStorageBackend().save( + ROIArchiveSourceMetadata.bind( + [ROI([PointShape(32.25, 40.5)], {"label": channel})], metadata + ), + path, + ) + paths.append(str(path)) + metadata_items.append(metadata) + filemanager = FakeFileManager() + metadata_handler = FakeMetadataHandler() + monkeypatch.setattr( + metadata_handler, + "get_pixel_size", + lambda _path: pytest.fail("Native explicit spacing must not be replaced"), + ) + handler = SimpleNamespace( + parser=SimpleNamespace( + parse_filename=lambda _filename: pytest.fail( + "Native metadata must not use a filename guess" + ) + ), + metadata_handler=metadata_handler, + ) + result = StreamingService(filemanager, handler, Path("/actual/source")).stream_rois( + RoiStreamingRequest( + viewer=FakeViewer(), + config=config_type(enabled=True), + status_callback=lambda _status: None, + error_callback=lambda error: pytest.fail(error), + roi_filenames=tuple(paths), + require_source_metadata=True, + ) + ) + assert result.streamed_paths == tuple(paths) + assert len(filemanager.saved_batches) == 2 + for (data, batch_paths, _backend, kwargs), expected in zip( + filemanager.saved_batches, metadata_items, strict=True + ): + stream = kwargs[ViewerStreamKwarg.STREAM_REQUEST.value] + metadata = ImagePayloadMetadata.from_viewer_image_metadata( + stream.source.item_fields[ViewerWireField.IMAGE_METADATA.value] + ) + assert metadata.source_voxel_spacing == expected.source_voxel_spacing + assert metadata.source_spatial_domain == expected.source_spatial_domain + assert ( + stream.source.metadata.metadata_by_path[batch_paths[0]] + == expected.source_component_metadata + ) + assert data[0][0].metadata == { + "label": expected.source_component_metadata["channel"] + } + assert data[0][0].shapes == [PointShape(32.25, 40.5)] + + +def test_explicit_native_reopening_rejects_an_external_roi_without_source(tmp_path): + from polystore.disk import DiskStorageBackend + + path = tmp_path / "A01_s001_w1_z001_t001.roi.zip" + DiskStorageBackend().save([ROI([PointShape(1, 2)], {"label": 1})], path) + filemanager = FakeFileManager() + with pytest.raises(ValueError, match="Native ROI source metadata is required"): + StreamingService(filemanager, SimpleNamespace(), tmp_path).stream_rois( + RoiStreamingRequest( + viewer=FakeViewer(), + config=NapariStreamingConfig(enabled=True), + status_callback=lambda _status: None, + error_callback=lambda error: pytest.fail(error), + roi_filenames=(str(path),), + require_source_metadata=True, + ) + ) + assert filemanager.saved_batches == [] + + def test_stream_rois_preserves_per_artifact_producer_identities(monkeypatch) -> None: monkeypatch.setattr( "polystore.roi.load_rois_from_zip", - lambda _path: [object()], + lambda _path: [ROI(shapes=[PointShape(1, 2)])], ) filemanager = FakeFileManager() config = FijiStreamingConfig(enabled=True) @@ -909,7 +1009,11 @@ def test_stream_rois_keeps_producer_identity_aligned_when_archive_is_empty( ) monkeypatch.setattr( "polystore.roi.load_rois_from_zip", - lambda path: [] if str(path).endswith("A01_empty.roi.zip") else [object()], + lambda path: ( + [] + if str(path).endswith("A01_empty.roi.zip") + else [ROI(shapes=[PointShape(1, 2)])] + ), ) filemanager = FakeFileManager() producer_identities = tuple( From 4f6f8733c91d43ff7f75d1b015c7bbf3ad9172f9 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 03:30:26 -0400 Subject: [PATCH 3/3] Record installed native viewer evidence and retained-plane roundtrip --- .../materialized_result_reopening_20260929.md | 56 +++++++++++++++++++ tests/unit/test_roi_source_metadata.py | 30 ++++++++++ 2 files changed, 86 insertions(+) diff --git a/paper/plans/materialized_result_reopening_20260929.md b/paper/plans/materialized_result_reopening_20260929.md index 34b22f4b3e..061c6da24b 100644 --- a/paper/plans/materialized_result_reopening_20260929.md +++ b/paper/plans/materialized_result_reopening_20260929.md @@ -55,3 +55,59 @@ PR #157 still owns worker_execution, runtime_exports and function_artifact_materialization. Do not compete there. This branch owns the independent shared directory consumer and native source-metadata persistence boundary. Full writer-success observation remains coordinated with #157. + +## Implemented native reopening checkpoint + +Production commit: 9f37cb66d. The source contract is intentionally extended; +this is not a proved-equivalent NRA migration. Expanded native class census: +162 original ClassDefs, 162 unique canonical joins, zero unprojected rows across +the original four modules plus ROI metadata, runtime metadata and materialization. +No global full-package/R1/native-MRO proof is claimed. + +- Query and streaming now use one path-policy-admitted explicit-directory inventory. + PlateFileStreamRequest owns result_directory; MCP derives its exposed parameter. +- Original plate context supplies ROI acquisition calibration, independently of + where outputs were saved. Explicit image-result reopening requires a verified + source receipt rather than manufacturing identity from a filename. +- Existing ImagePayloadMetadata is projected into the native ROI sidecar, and + decoded through its canonical codec on reopening. Source schema/fields and + geometry remain owned by their existing types; no extra format registry exists. + Missing or conflicting source declarations fail the explicit native route. +- Native per-archive metadata drives components, retained plane projection, + domain and spacing. Existing partition_indices groups wire-compatible batches; + existing producer.for_indices preserves identity after empty-archive omissions. + One source-owned calibrated_metadata method fills absent spacing for images + and ROIs without replacing explicit native spacing. +- The stream capability now truthfully declares viewer launch/update side effects. + +Focused source checks passed 156; installed checks passed 157, including the added +retained-plane native round trip. Application module paths were verified inside +the owned installed venv, with source used only for test fixtures. A first new +plane test mistakenly expected integer display-coordinate values; source tracing +confirmed the existing projection deliberately returns canonical strings. The +test now checks that original declaration's exact output and round-trip equality; +no production codec or assertion was relaxed to hide a serialization change. +Two pytest configuration warnings are due to deliberately disabled async plugins. + +Installed public MCP on isolated :90/5691 reopened both a fractional POINT archive +and a real native-materializer polygon archive from retained_native_20260929. +Despite B99/w7 artifact names, A01/channel 1 source identity, shape 128x128, +XY scale 0.65, zero translation and exact native labels/coordinates were retained. +All six same-coordinate raw/result/combined captures under [444,26058] and +[1500,9000], gamma 1, were personally opened. Canvas 1440x950, camera center +(0,41.275,41.275), zoom 5.366586538461538 and axes stayed fixed. Exact raw 4x4 +values were also read via MCP. These are manual synthetic transport fixtures, +NOT detections: accept transport, reject any biological accuracy claim. + +Complete native envelopes, requests and disposition are retained under +/home/ts/wt/openhcs-live-point-evidence-20260929/native-reopening-20260929. +MCP closed the extra viewer and a listener check confirmed port 5691 absent. +Current H002 processes stayed alive; SCIENCE_SHA256SUMS passed unchanged. +No frozen rerun, tuning, artifact rewrite or new reference/held-out access occurred. + +Still OPEN: source-bearing runtime writer-success admission with #157; the real +compile/run-to-reopen journey; truthful reopening of historical H002 files whose +source header was never persisted; full 3D/orthogonal installed viewer proof; +fresh context-isolated blind QA and scientific acceptance. Keep issue #134 and +the full goal active. Do not substitute this synthetic checkpoint for them. +Resource headroom remains warning, so no extra agent fleet was started. diff --git a/tests/unit/test_roi_source_metadata.py b/tests/unit/test_roi_source_metadata.py index 2b1fff511c..10dbabf88b 100644 --- a/tests/unit/test_roi_source_metadata.py +++ b/tests/unit/test_roi_source_metadata.py @@ -9,6 +9,8 @@ from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.core.runtime_object_labels import ( ObjectLabelPayload, ObjectLabelVariantData, @@ -63,6 +65,34 @@ def test_source_metadata_is_optional_only_for_external_shapes(): ROIArchiveSourceMetadata.decode([malformed]) +def test_native_sidecar_preserves_retained_source_planes(tmp_path): + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.SOURCE_BINDING, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/source/z1.tif", "/source/z2.tif"), + component_metadata=( + {"well": "B02", "z_index": 1}, + {"well": "B02", "z_index": 2}, + ), + ), + source_voxel_spacing=SourceVoxelSpacing((2.0, 0.65, 0.65)), + ) + rois = [ROI([PointShape(3.25, 4.5)], {"label": 12, "plane_indices": (1,)})] + path = tmp_path / "unrelated-name.roi.zip" + DiskStorageBackend().save(ROIArchiveSourceMetadata.bind(rois, metadata), path) + restored = load_rois_from_zip(path) + decoded = ROIArchiveSourceMetadata.decode(restored) + assert decoded.source_provenance == metadata.source_provenance + assert metadata.retained_plane_component_values() == {"z_index": ("1", "2")} + assert ( + decoded.retained_plane_component_values() + == metadata.retained_plane_component_values() + ) + assert decoded.plane_axis is RuntimePlaneAxis.SOURCE_BINDING + assert decoded.source_voxel_spacing == metadata.source_voxel_spacing + assert ROIArchiveSourceMetadata.geometry(restored) == rois + + def test_real_roi_materialization_persists_output_source_metadata(tmp_path): labels = np.zeros((8, 8), dtype=np.int32) labels[2:6, 3:7] = 12