diff --git a/openhcs/processing/backends/cellprofiler/save_images.py b/openhcs/processing/backends/cellprofiler/save_images.py index 63065380ba..796419fcbf 100644 --- a/openhcs/processing/backends/cellprofiler/save_images.py +++ b/openhcs/processing/backends/cellprofiler/save_images.py @@ -22,6 +22,7 @@ MaterializationSourceIdentityRelation, MeasurementsArtifactType, ) +from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.image_file_serialization import image_payload_as_uint8 from openhcs.core.memory import numpy from openhcs.core.measurement_row_materialization import ( @@ -66,9 +67,11 @@ ProcessingContract, ) from openhcs.processing.materialization import ( + ExecutionAxisMaterializationRelativePathScope, ImageFileOptions, MaterializationSpec, MaterializedFilenameIdentity, + WriteMode, ) if TYPE_CHECKING: @@ -358,6 +361,7 @@ class SaveImagesModule( function_variants = ("save_images_with_measurements",) validated = True confidence = 1.0 + relative_path_scope = ExecutionAxisMaterializationRelativePathScope() image_kind_setting = SettingNameFamily("Select the type of image to save") source_image_setting = SettingNameFamily("Select the image to save") @@ -892,8 +896,14 @@ def _materialization_spec( filename_suffix=f"{suffix}{file_format.value}", filename_identity=filename_identity, relative_path_template=relative_path_template, + relative_path_scope=cls.relative_path_scope, ) - materialization = MaterializationSpec(options) + write_mode = ( + WriteMode.OVERWRITE + if _bool_setting(module, cls.overwrite_setting, True) + else WriteMode.ERROR + ) + materialization = MaterializationSpec(options, write_mode=write_mode) materialization.candidate_paths(f"{output_name}.pkl") return materialization @@ -921,6 +931,7 @@ def _recorded_save_images_rows( filename_suffix: str, file_format: SaveImagesFileFormat, output_location: str | None, + context: ProcessingContext | None, ) -> DataclassMeasurementColumnarRows: suffix = filename_suffix if append_suffix else "" if filename_method is SaveImagesFilenameMethod.FROM_IMAGE_FILENAME: @@ -937,7 +948,12 @@ def _recorded_save_images_rows( ) else: filename = f"{single_file_name}{suffix}{file_format.value}" - relative_path = _relative_template(output_location, filename) + relative_path = str( + SaveImagesModule.relative_path_scope.project( + PurePosixPath(_relative_template(output_location, filename)), + context, + ) + ) pathname = PurePosixPath(relative_path).parent.as_posix() if pathname == ".": pathname = "" @@ -983,6 +999,7 @@ def save_images( base_image_folder: str | None = None, series_axis: SaveImagesSeriesAxis = SaveImagesSeriesAxis.TIMEPOINT, lossless_compression: bool = True, + context: ProcessingContext | None = None, ) -> tuple[RuntimeArrayData, RuntimeArrayData]: """Prepare a selected image or object set for saving without replacing the pipeline image. @@ -1004,6 +1021,7 @@ def save_images( base_image_folder, series_axis, lossless_compression, + context, ) return image, _converted_saved_image( image_to_save, @@ -1038,6 +1056,7 @@ def save_images_with_measurements( series_axis: SaveImagesSeriesAxis = SaveImagesSeriesAxis.TIMEPOINT, lossless_compression: bool = True, slice_index: int = 0, + context: ProcessingContext | None = None, ) -> tuple[RuntimeArrayData, RuntimeArrayData, DataclassMeasurementColumnarRows]: """Prepare an image for saving and record its output-file measurements. @@ -1069,6 +1088,7 @@ def save_images_with_measurements( filename_suffix=filename_suffix, file_format=file_format, output_location=output_location, + context=context, ) return image, converted, rows diff --git a/openhcs/processing/materialization/__init__.py b/openhcs/processing/materialization/__init__.py index 1cf8d2a62b..a8e58d6dde 100644 --- a/openhcs/processing/materialization/__init__.py +++ b/openhcs/processing/materialization/__init__.py @@ -29,6 +29,11 @@ TextOptions, TiffStackOptions, ) +from openhcs.processing.materialization.path_scopes import ( + ExecutionAxisMaterializationRelativePathScope, + MaterializationRelativePathScope, + SharedMaterializationRelativePathScope, +) from openhcs.processing.materialization.presets import ( csv_dataclass_materializer, csv_materializer, @@ -76,4 +81,7 @@ "tiff_stack", "text_only", "tabular_field_names_from_materialization", + "MaterializationRelativePathScope", + "SharedMaterializationRelativePathScope", + "ExecutionAxisMaterializationRelativePathScope", ] diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index c2205ef136..6feb2a7bb0 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -2444,9 +2444,14 @@ def replace_backreference(match: re.Match[str]) -> str: options.relative_path_template, ) expanded = _format_image_sequence_index(expanded, sequence_index) + relative_path = _normalized_relative_materialization_path(expanded) + scoped_relative_path = options.relative_path_scope.project( + relative_path, + context.context, + ) return _relative_materialization_output_path( context.base_path, - _normalized_relative_materialization_path(expanded), + _normalized_relative_materialization_path(str(scoped_relative_path)), ) diff --git a/openhcs/processing/materialization/options.py b/openhcs/processing/materialization/options.py index d93a4e4a45..d29b4e8f5f 100644 --- a/openhcs/processing/materialization/options.py +++ b/openhcs/processing/materialization/options.py @@ -13,6 +13,10 @@ from typing import Any, Callable, Dict, List, Optional from openhcs.core.runtime_measurements import RuntimeMeasurementFeature +from openhcs.processing.materialization.path_scopes import ( + MaterializationRelativePathScope, + SharedMaterializationRelativePathScope, +) class MaterializedFilenameIdentity(str, Enum): @@ -174,6 +178,9 @@ class ImageFileOptions(FileOutputOptions, SourceOptions): """One image file written through the registered image format family.""" relative_path_template: str | None = None + relative_path_scope: MaterializationRelativePathScope = field( + default_factory=SharedMaterializationRelativePathScope + ) @dataclass(frozen=True) diff --git a/openhcs/processing/materialization/path_scopes.py b/openhcs/processing/materialization/path_scopes.py new file mode 100644 index 0000000000..85e9b481e8 --- /dev/null +++ b/openhcs/processing/materialization/path_scopes.py @@ -0,0 +1,56 @@ +"""Nominal relative-path scopes for materialized artifacts.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openhcs.core.context.processing_context import ProcessingContext + + +class MaterializationRelativePathScope(ABC): + """Project a writer-declared relative path into its runtime scope.""" + + @abstractmethod + def project( + self, + relative_path: PurePosixPath, + context: ProcessingContext | None, + ) -> PurePosixPath: + """Return the relative path owned by one runtime execution context.""" + + +@dataclass(frozen=True, slots=True) +class SharedMaterializationRelativePathScope(MaterializationRelativePathScope): + """Keep a relative output path shared across execution axes.""" + + def project( + self, + relative_path: PurePosixPath, + context: ProcessingContext | None, + ) -> PurePosixPath: + del context + return relative_path + + +@dataclass(frozen=True, slots=True) +class ExecutionAxisMaterializationRelativePathScope(MaterializationRelativePathScope): + """Isolate relative outputs when one execution contains multiple axes.""" + + def project( + self, + relative_path: PurePosixPath, + context: ProcessingContext | None, + ) -> PurePosixPath: + if context is None or context.execution_runtime is None: + return relative_path + if len(context.execution_runtime.execution_axis_values) <= 1: + return relative_path + + from openhcs.core.source_projection import OpenHCSPlaneAddress + + axis_token = OpenHCSPlaneAddress.component_token(context.require_axis_id()) + return PurePosixPath(axis_token) / relative_path diff --git a/tests/unit/test_cellprofiler_export_measurement_outputs.py b/tests/unit/test_cellprofiler_export_measurement_outputs.py index 951648965a..1617ca8a64 100644 --- a/tests/unit/test_cellprofiler_export_measurement_outputs.py +++ b/tests/unit/test_cellprofiler_export_measurement_outputs.py @@ -33,6 +33,7 @@ from openhcs.core.runtime_measurements import MeasurementTable from openhcs.core.runtime_measurements import ( MeasurementRowAxisField, + MeasurementRowValueField, MeasurementScope, ) from openhcs.core.runtime_stores import RuntimeArtifactBatch, RuntimeValueStore @@ -341,6 +342,33 @@ def test_save_images_file_measurement_output_and_rows_are_conditional() -> None: 3 } + multi_axis_context = ProcessingContext(axis_id="A01") + multi_axis_context.execution_runtime = SimpleNamespace( + execution_axis_values=("A01", "A02") + ) + _returned_main, _saved, scoped_rows = save_images_with_measurements( + image, + image_to_save=image, + saved_image_name="DNA", + filename_method=SaveImagesFilenameMethod.SINGLE_NAME, + single_file_name="SavedDNA", + file_format=SaveImagesFileFormat.PNG, + output_location="exports", + slice_index=3, + context=multi_axis_context, + ) + scoped_row_values = { + row[MeasurementRowAxisField.FEATURE_NAME.value]: row[ + MeasurementRowValueField.RESULT_VALUE.value + ] + for row in scoped_rows.iter_row_mappings() + } + assert scoped_row_values == { + "FileName_DNA": "SavedDNA.png", + "PathName_DNA": "A01/exports", + "URL_DNA": "file:A01/exports/SavedDNA.png", + } + measurement_plan = ArtifactOutputPlan( name=measurement.name, path="/memory/save_images_measurements.pkl", diff --git a/tests/unit/test_cellprofiler_save_images.py b/tests/unit/test_cellprofiler_save_images.py index 71bb29747d..a6162bab2b 100644 --- a/tests/unit/test_cellprofiler_save_images.py +++ b/tests/unit/test_cellprofiler_save_images.py @@ -3,6 +3,8 @@ from __future__ import annotations import inspect +from pathlib import PurePosixPath +from types import SimpleNamespace import numpy as np import pytest @@ -20,6 +22,7 @@ SourceStackLineageSourceRelation, ) from openhcs.core.callable_contract import CallableContract +from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import ( FunctionInvocationKey, compile_function_pattern, @@ -77,9 +80,11 @@ ProcessingContract, ) from openhcs.processing.materialization import ( + ExecutionAxisMaterializationRelativePathScope, ImageFileOptions, MaterializationSpec, MaterializedFilenameIdentity, + WriteMode, materialize, ) from polystore.filemanager import FileManager @@ -664,6 +669,98 @@ def test_save_images_projects_sequential_numbers_from_materialization_order() -> ) +def test_save_images_isolates_declared_relative_paths_for_multi_axis_execution() -> ( + None +): + module = _module( + **{ + "Select the image to save": "ImageToSave", + "Select method for constructing file names": "Sequential numbers", + "Enter single file name": "CroppedFlyImage", + "Number of digits": "4", + "Append a suffix to the image file name?": "Yes", + "Text to append to the image name": "RGB", + "Saved file format": "tiff", + "Overwrite existing files without warning?": "No", + } + ) + contract = _contract(module) + materialization = contract.artifact_outputs[0].materialization + assert isinstance(materialization, MaterializationSpec) + options = _image_file_options(contract) + assert isinstance( + options.relative_path_scope, + ExecutionAxisMaterializationRelativePathScope, + ) + assert materialization.write_mode is WriteMode.ERROR + + context = ProcessingContext(axis_id="A01") + context.execution_runtime = SimpleNamespace(execution_axis_values=("A01", "A02")) + filemanager = FileManager({"memory": MemoryStorageBackend()}) + primary_path = materialize( + materialization, + RuntimeSliceAlignedValues( + tuple( + ImagePayloadMetadata().payload_with( + np.full((3, 4), item, dtype=np.uint8), + None, + ) + for item in (1, 2, 3) + ) + ), + "/analysis/SaveImages.pkl", + filemanager, + ("memory",), + context=context, + ) + + assert primary_path == "/analysis/A01/CroppedFlyImage0001RGB.tiff" + for item in (1, 2, 3): + np.testing.assert_array_equal( + filemanager.load( + f"/analysis/A01/CroppedFlyImage{item:04d}RGB.tiff", + "memory", + ), + np.full((3, 4), item, dtype=np.uint8), + ) + + +def test_save_images_keeps_declared_relative_paths_for_single_axis_execution() -> None: + scope = ExecutionAxisMaterializationRelativePathScope() + context = ProcessingContext(axis_id="A01") + context.execution_runtime = SimpleNamespace(execution_axis_values=("A01",)) + + assert scope.project(PurePosixPath("exports/Saved.tiff"), context) == ( + PurePosixPath("exports/Saved.tiff") + ) + + +def test_save_images_overwrite_setting_owns_materialization_write_mode() -> None: + overwrite_contract = _contract( + _module( + **{ + "Select the image to save": "ImageToSave", + "Overwrite existing files without warning?": "Yes", + } + ) + ) + refuse_contract = _contract( + _module( + **{ + "Select the image to save": "ImageToSave", + "Overwrite existing files without warning?": "No", + } + ) + ) + + assert overwrite_contract.artifact_outputs[0].materialization.write_mode is ( + WriteMode.OVERWRITE + ) + assert refuse_contract.artifact_outputs[0].materialization.write_mode is ( + WriteMode.ERROR + ) + + def test_save_images_from_image_filename_uses_source_identity_and_suffix() -> None: module = _module( **{