From 3d377842cdb6e2861fb07c46303244fd9a6c1561 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 14:47:28 -0400 Subject: [PATCH 01/13] K7: receipt re-measured at d597aa224 Co-Authored-By: Claude Opus 5.5 --- .../K7-execution-protocol.md | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/refactor/consolidation-20261009/K7-execution-protocol.md diff --git a/docs/refactor/consolidation-20261009/K7-execution-protocol.md b/docs/refactor/consolidation-20261009/K7-execution-protocol.md new file mode 100644 index 0000000000..9868df32b9 --- /dev/null +++ b/docs/refactor/consolidation-20261009/K7-execution-protocol.md @@ -0,0 +1,53 @@ +# K7: Execution protocol + +**Head audited:** `openhcs` `main` at `d597aa224` (#1205; G7, K2, K3, N1 merged). **Rules:** [00-RULES.md](00-RULES.md). **Step 4.** +**Shared abstractions** ([02-SHARED-ABSTRACTIONS.md](02-SHARED-ABSTRACTIONS.md)). *Builds:* none. *Uses:* `AutoRegisterMeta`, the G1 axis family. + +Re-measured at this head. G7 replaced part of K7 (see Already done); the other three items were still open. + +## Already done (closure note for the worker-lane item) + +**Worker lanes dispatching tuple commands on strings with two receivers: landed in G7 (#1191).** Verified at `d597aa224`: `core/orchestrator/worker_execution.py:655-760` declares `WorkerCommand.serve` with `AnsweredWorkerCommand` (`PrepareWorker`, `ExecuteInheritedLane`, `ExecuteShippedLane`, `MapPartitions`, `FinishExecution`) and `StopWorker`; replies are `WorkerReply.settle` (`WorkerResult`, `WorkerFailure`, `WorkerProgress`); one entry `_serve_worker` and one `WorkerProcess` reader. No tag literal is sent (`tests/unit/test_g7_runtime_protocol_guards.py`). Nothing remains for K7 here. + +## What is wrong + +**Three execution-protocol values encode their variants as `None` and flags, so every consumer re-decides the variant: the component-group scope, the processing context's phase, and the execution result.** + +1. **`ComponentGroupScope` packs three variants into two fields** (`core/component_group_scope.py:30`). `keys=(None,), component=None` is ungrouped, `keys=(None,), component=X` is dynamic (keys discovered at run time), anything else is static. The variant is re-decided through `is_ungrouped`/`is_grouped`/`is_dynamic`/`is_static`/`has_single_static_key`/`require_single_static_key`: **73 predicate reads** (AST scan, 25 inside the class, 48 in 8 consumer modules: `pipeline/path_planner.py` 20, `artifacts/input_projection.py` 9, `runtime_stores.py` 8, `artifacts/plans.py` 6, `pipeline/compiler.py` 2, `steps/function_execution.py` 2, `runtime_artifact_values.py` 1), plus `component is None` checks on scopes in `compiler.py:1213,1223`. + - The same two fields are restated on every `ArtifactPlan` as `group_keys` and `group_component` (`artifacts/plans.py:288-289`), re-packed into a scope by `group_scope()` on each call and unpacked again by `PathPlannerGroupScope.from_output_plan/from_plan`, `RuntimeArtifactQuery.from_input_plan` and `ArtifactInputProjectionPlan.__post_init__` (`from_raw(scope.keys, component=scope.component)` written 8 times). +2. **`RuntimeValue.compose(producer_group_scope=None)` is a `None` mode** (`runtime_artifact_values.py:108`). `None` means "plain composition"; a scope means "reconstruct the producer's group axis". Each kind's `compose_runtime_values` (`artifacts/kinds.py:383,836,1094,1301,1385,1510`) re-decides it: three test `producer_group_scope is None and len(values) == 1`, three `del` the argument. +3. **`ProcessingContext` mixes compile-time and execution-time state** (`core/context/processing_context.py:64`, 324 lines, 258 references in 45 production files). One mutable object carries the compiler's facts (`step_plans`, `axis_id`, `filemanager`, sequential combinations, visualizers, axis filters) and the executor's (`runtime_value_store` and four runtime caches, `completed_step_outputs`, `execution_runtime`, `debug_event_sink`, the per-step value scope). The phase is a flag: `_is_frozen`, checked by `is_frozen()` in 6 compiler stages (`compiler.py:252,536,660,738,775,800`) and in `worker_execution.py:1473`. The compiled context is pickled to workers with empty runtime stores, executed in place, and its stores cleared afterwards (`worker_execution.py:1357`). `compiler.py:227` sets `context.visualizer_config`, an attribute nothing declares or reads. +4. **`ExecutionResult` is a longhand union** (`orchestrator/execution_result.py:190`): an `ExecutionStatus` enum plus `failed_combination` and `error_message` Optionals meaningful only for some statuses, three `is_*` predicates, and three constructors that restate the field defaults. `RuntimeObservationMode` (same file) is an enum whose members' behaviour is written as `if self is …` branches (rule 1a). + +## Target + +- **Scope variants.** `ComponentGroupScope` is an ABC with `UngroupedScope`, and `GroupedScope` (holds `component`) with `DynamicGroupScope` and `StaticGroupScope` (holds `keys`). Each variant owns `runtime_keys`, `contains_runtime_key`, `contains_scope`, `resolve_runtime_key`, `select_runtime_key`, `excludes`, `output_lineage_scope` and the other questions consumers asked through predicates. `ComponentGroupScope.of(keys, component)` is the one constructor from raw values. The predicates and `from_raw` are deleted. +- **Plans hold a scope.** `ArtifactPlan.group_scope` is a field; `group_keys`/`group_component` and every unpack/re-pack are deleted. +- **Two compositions.** `RuntimeValue.compose(values)` and `RuntimeValue.compose_groups(values, producer_scope: GroupedScope)`; kinds override `compose_runtime_groups` where reconstruction differs. The `producer_group_scope` parameter is deleted. +- **Phase types.** `ProcessingContext` is the base carrying compile facts. `CompilingContext` is the only mutable one (`inject_plan`, `freeze() -> CompiledContext`); `CompiledContext` is immutable and is what bundles and workers carry; `ExecutingContext(CompiledContext)` adds the runtime stores, caches, debug sink, lane runtime and completed outputs, and is built per execution by `CompiledContext.executing(...)`. `_is_frozen`, `is_frozen()`, the six compiler checks and `visualizer_config` are deleted. +- **One result family.** `ExecutionResult` ABC with `ExecutionSucceeded`, `ExecutionFailed` (owns `error_message`, `failed_combination`) and `ExecutionCancelled` (owns `error_message`); `ExecutionStatus` and the `is_*` predicates are deleted. `RuntimeObservationMode` becomes a family whose members own `retain_records`. + +## Guards + +`tests/unit/test_k7_execution_protocol_guards.py`: no production read of the deleted scope predicates or `from_raw`; no `group_keys`/`group_component` plan fields; no `producer_group_scope` parameter; no `is_frozen`/`_is_frozen`; no `ExecutionStatus` in `execution_result.py`; the scope, context and result families are exactly the declared members. + +## Tests + +Touched tests take the new constructors and types. Run: the guards; the unit test files that import the changed modules (one process each); official30 parity (30/30). + +## New-case experiments + +- A new scope variant today needs a new `(keys, component)` encoding and a branch at each of 48 predicate reads. After: one subclass implementing the family's methods. +- A new execution outcome today needs an enum member, a predicate, a constructor and optional fields. After: one `ExecutionResult` subclass. + +## Done when + +The guards pass; touched tests pass; official30 is 30/30; the structural ratchet and nra-r1 show zero regressions. + +## Progress + +- [x] Receipt. +- [ ] ExecutionResult and RuntimeObservationMode families. +- [ ] ComponentGroupScope variants, plan `group_scope` field, compose split. +- [ ] Context phase types. +- [ ] Guards, tests, ratchet, parity, PR. From 0c376e01c93ac490e52c8094b28e1502a4dfeef7 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 14:53:02 -0400 Subject: [PATCH 02/13] WIP: K7 execution result and observation-mode families Co-Authored-By: Claude Opus 5.5 --- .../K7-execution-protocol.md | 2 +- openhcs/core/orchestrator/bundle_execution.py | 17 +- openhcs/core/orchestrator/execution_result.py | 217 +++++++++--------- openhcs/core/orchestrator/orchestrator.py | 2 +- openhcs/core/orchestrator/worker_execution.py | 53 ++--- openhcs/core/orchestrator/worker_lanes.py | 2 +- openhcs/runtime/zmq_execution_observation.py | 30 +-- openhcs/runtime/zmq_execution_signature.py | 2 +- openhcs/runtime/zmq_worker_execution.py | 2 +- .../test_chained_callable_lineage_journey.py | 6 +- .../test_image_math_publication_620.py | 6 +- .../test_measurement_declaration_journey.py | 6 +- ...rimary_segmentation_diagnostics_journey.py | 6 +- ...est_source_schema_aggregate_publication.py | 6 +- tests/unit/test_compiled_execution.py | 41 ++-- ..._completed_output_publication_lifecycle.py | 10 +- tests/unit/test_deleted_core_names_absent.py | 4 +- .../test_function_artifact_materialization.py | 9 +- .../test_function_step_execution_scope.py | 14 +- .../test_materialization_batch_outcomes.py | 6 +- ...est_nonpersistent_viewer_state_evidence.py | 10 +- tests/unit/test_openhcs_measured_run.py | 8 +- .../test_orchestrator_execution_result.py | 15 +- tests/unit/test_orchestrator_lane_planning.py | 55 ++--- .../unit/test_runtime_execution_validation.py | 6 +- tests/unit/test_well_throughput_scaling.py | 6 +- tests/unit/test_zmq_execution_server.py | 39 ++-- tests/unit/test_zmq_runtime_outcome_export.py | 22 +- 28 files changed, 306 insertions(+), 296 deletions(-) diff --git a/docs/refactor/consolidation-20261009/K7-execution-protocol.md b/docs/refactor/consolidation-20261009/K7-execution-protocol.md index 9868df32b9..29022ee775 100644 --- a/docs/refactor/consolidation-20261009/K7-execution-protocol.md +++ b/docs/refactor/consolidation-20261009/K7-execution-protocol.md @@ -47,7 +47,7 @@ The guards pass; touched tests pass; official30 is 30/30; the structural ratchet ## Progress - [x] Receipt. -- [ ] ExecutionResult and RuntimeObservationMode families. +- [x] ExecutionResult and RuntimeObservationMode families (`ExecutionSucceeded`/`ExecutionFailed`/`ExecutionCancelled`; `ReturnNoRecords`/`ReturnAllRecords`/`ReturnDatasetInputRecords`; `ZMQExecutionAxisOutcome` deleted, outcome export stores results without records). - [ ] ComponentGroupScope variants, plan `group_scope` field, compose split. - [ ] Context phase types. - [ ] Guards, tests, ratchet, parity, PR. diff --git a/openhcs/core/orchestrator/bundle_execution.py b/openhcs/core/orchestrator/bundle_execution.py index 1dba0722e7..48c43136f9 100644 --- a/openhcs/core/orchestrator/bundle_execution.py +++ b/openhcs/core/orchestrator/bundle_execution.py @@ -41,6 +41,7 @@ from openhcs.core.orchestrator.cancellation import ExecutionCancelledError from openhcs.core.orchestrator.execution_result import ( ExecutionResult, + ReturnAllRecords, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -144,7 +145,7 @@ def is_success(self) -> bool: def require_success(self) -> None: """Reject failed axes at terminal boundaries without dropping their outputs.""" failures = [ - f"{axis_id}: {result.status.value}: {result.error_message or 'no error detail'}" + f"{axis_id}: {result.describe()}" for axis_id, result in self.items() if not result.is_success() ] @@ -168,7 +169,7 @@ class BundleExecutionRequest(ProgressExecutionContext): visualizer: ExecutionVisualizerABC | None log_file_base: Optional[str] progress_queue: ProgressQueue - runtime_observation_mode: RuntimeObservationMode + runtime_observation_mode: type[RuntimeObservationMode] debug_execution_policy: DebugExecutionPolicy prepared_worker_runner: PreparedForkWorkerLaneRunner | None = None @@ -283,7 +284,7 @@ def execute_bundle_request( } ) if any( - result.is_cancelled() for result in execution_results.values() + result.stops_lane for result in execution_results.values() ): raise ExecutionCancelledError( "Execution cancelled during worker execution" @@ -393,12 +394,12 @@ def validate_bundle_execution( dataset_step_indexes = validate_dataset_scoped_contexts(compiled_contexts) if ( dataset_step_indexes - and request.runtime_observation_mode is RuntimeObservationMode.OMIT + and not request.runtime_observation_mode.collects_records ): raise ValueError( - "Dataset-scoped FunctionSteps require RuntimeObservationMode." - "MERGE_INTO_PARENT so compiled artifact inputs are available after " - "worker execution." + "Dataset-scoped FunctionSteps require a runtime observation mode " + "that returns records, so compiled artifact inputs are available " + "after worker execution." ) runtime_environment = request.execution_bundle.runtime_environment return ValidatedBundleExecution( @@ -654,7 +655,7 @@ def execute_dataset_scoped_steps( context_key=context_key, context=context, records=records, - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, outputs=StepExecutionObservation.combine( observations_by_context[context_key] ), diff --git a/openhcs/core/orchestrator/execution_result.py b/openhcs/core/orchestrator/execution_result.py index 157ca6db78..67205c4830 100644 --- a/openhcs/core/orchestrator/execution_result.py +++ b/openhcs/core/orchestrator/execution_result.py @@ -6,13 +6,21 @@ """ import copyreg -from dataclasses import dataclass, field -from enum import Enum +from abc import ABC, abstractmethod +from dataclasses import dataclass, field, replace from types import MappingProxyType -from typing import TYPE_CHECKING, Mapping, Optional +from typing import TYPE_CHECKING, ClassVar, Mapping from openhcs.constants.constants import Backend from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.progress.types import ( + AxisErrorProgressPhase, + CancelledProgressPhase, + CancelledProgressStatus, + ErrorProgressStatus, + ProgressPhase, + ProgressStatus, +) from openhcs.core.runtime_stores import RuntimeArtifactQuery, StoredRuntimeValue from openhcs.core.steps.abstract import StepExecutionObservation @@ -46,14 +54,6 @@ def register(cls) -> None: RuntimeExecutionTransportSerialization.register() -class ExecutionStatus(Enum): - """Status of pipeline execution for an axis or combination.""" - - SUCCESS = "success" - ERROR = "error" - CANCELLED = "cancelled" - - @dataclass(frozen=True) class RuntimeContextObservation: """Runtime records produced for one compiled execution context.""" @@ -69,7 +69,7 @@ def from_context( context_key: str, context: ProcessingContext, records: tuple[StoredRuntimeValue, ...], - runtime_observation_mode: "RuntimeObservationMode", + runtime_observation_mode: type["RuntimeObservationMode"], outputs: StepExecutionObservation, ) -> "RuntimeContextObservation": """Retain requested records beside the completed output projections.""" @@ -110,52 +110,66 @@ def merge_into(self, execution_contexts: Mapping[str, ProcessingContext]) -> Non context.record_completed_step_outputs(context_observation.outputs) -class RuntimeObservationMode(Enum): - """Controls whether worker runtime records are returned to the parent.""" - - MERGE_INTO_PARENT = "merge_into_parent" - MERGE_DATASET_INPUTS = "merge_dataset_inputs" - OMIT = "omit" - - @property - def collects_records(self) -> bool: - return self is not RuntimeObservationMode.OMIT - - @classmethod - def from_parent_requirement(cls, required: bool) -> "RuntimeObservationMode": - """Select the mode implied by compiled parent-side execution needs.""" +class RuntimeObservationMode(ABC): + """Which worker runtime records an execution returns to the parent.""" - return cls.MERGE_INTO_PARENT if required else cls.OMIT + collects_records: ClassVar[bool] = True + returns_every_record: ClassVar[bool] = False @classmethod def for_compiled_bundle( cls, bundle: "CompiledExecutionBundle" - ) -> "RuntimeObservationMode": + ) -> type["RuntimeObservationMode"]: """Consolidation carries rendered tables; only dataset-scoped inputs need records.""" if bundle.requires_parent_runtime_observation: - return cls.MERGE_DATASET_INPUTS - return cls.OMIT + return ReturnDatasetInputRecords + return ReturnNoRecords + @classmethod def including_parent_requirement( - self, + cls, required: bool, - ) -> "RuntimeObservationMode": + ) -> type["RuntimeObservationMode"]: """Return this mode strengthened by an additional retention requirement.""" - return type(self).MERGE_INTO_PARENT if required else self + return ReturnAllRecords if required else cls + @classmethod + @abstractmethod def retain_records( - self, + cls, records: tuple[StoredRuntimeValue, ...], context: ProcessingContext, ) -> tuple[StoredRuntimeValue, ...]: """Keep records consumed by the selected parent-side execution route.""" - if self is RuntimeObservationMode.OMIT: - return () - if self is RuntimeObservationMode.MERGE_INTO_PARENT: - return records + +class ReturnNoRecords(RuntimeObservationMode): + """Return no runtime records; the parent needs none.""" + + collects_records = False + + @classmethod + def retain_records(cls, records, context): + return () + + +class ReturnAllRecords(RuntimeObservationMode): + """Return every runtime record to the parent.""" + + returns_every_record = True + + @classmethod + def retain_records(cls, records, context): + return records + + +class ReturnDatasetInputRecords(RuntimeObservationMode): + """Return only the records dataset-scoped steps consume in the parent.""" + + @classmethod + def retain_records(cls, records, context): selected_addresses = set() for plan in context.step_plans.values(): if not plan.execution_scope.requires_parent_runtime_observation: @@ -187,87 +201,74 @@ def retain_records( @dataclass(frozen=True) -class ExecutionResult: - """ - Typed result of pipeline execution for a single axis. - - Replaces dict-based results to provide compile-time type safety - and explicit contracts. +class ExecutionResult(ABC): + """Outcome of executing one axis value's compiled contexts.""" - Attributes: - status: Execution status (success, error, etc.) - axis_id: Identifier for the axis that was executed - failed_combination: Optional key of the combination that failed (for sequential mode) - error_message: Optional error message if status is ERROR - """ + stops_lane: ClassVar[bool] = False - status: ExecutionStatus axis_id: str - failed_combination: Optional[str] = None - error_message: Optional[str] = None runtime_observation: RuntimeExecutionObservation = field( default_factory=RuntimeExecutionObservation ) def is_success(self) -> bool: - """Check if execution was successful.""" - return self.status == ExecutionStatus.SUCCESS + return False - def is_error(self) -> bool: - """Check if execution failed.""" - return self.status == ExecutionStatus.ERROR + def describe(self) -> str: + """Return this outcome as one line of text.""" + return type(self).__name__ - def is_cancelled(self) -> bool: - return self.status == ExecutionStatus.CANCELLED + def without_runtime_observation(self) -> "ExecutionResult": + """Return this outcome without the worker runtime records it carries.""" + return replace(self, runtime_observation=RuntimeExecutionObservation()) - @classmethod - def success( - cls, - axis_id: str, - runtime_observation: RuntimeExecutionObservation | None = None, - ) -> "ExecutionResult": - """Create a successful execution result.""" - return cls( - status=ExecutionStatus.SUCCESS, - axis_id=axis_id, - runtime_observation=runtime_observation or RuntimeExecutionObservation(), - ) - @classmethod - def error( - cls, - axis_id: str, - failed_combination: Optional[str] = None, - error_message: Optional[str] = None, - runtime_observation: RuntimeExecutionObservation | None = None, - ) -> "ExecutionResult": - """Create an error execution result.""" - return cls( - status=ExecutionStatus.ERROR, - axis_id=axis_id, - failed_combination=failed_combination, - error_message=error_message, - runtime_observation=( - RuntimeExecutionObservation() - if runtime_observation is None else runtime_observation - ), - ) +@dataclass(frozen=True) +class ExecutionSucceeded(ExecutionResult): + """Every compiled context of the axis ran to completion.""" - @classmethod - def cancelled( - cls, - axis_id: str, - *, - error_message: str, - runtime_observation: RuntimeExecutionObservation | None = None, - ) -> "ExecutionResult": - """Return completed outputs from a cooperative cancellation boundary.""" - return cls( - status=ExecutionStatus.CANCELLED, - axis_id=axis_id, - error_message=error_message, - runtime_observation=( - RuntimeExecutionObservation() - if runtime_observation is None else runtime_observation - ), + def is_success(self) -> bool: + return True + + +@dataclass(frozen=True, kw_only=True) +class ExecutionStopped(ExecutionResult): + """Execution of the axis ended before completion.""" + + progress_phase: ClassVar[type[ProgressPhase]] + progress_status: ClassVar[type[ProgressStatus]] + + error_message: str + failed_combination: str | None = None + + def describe(self) -> str: + return f"{type(self).__name__}: {self.error_message}" + + def at_combination( + self, + context_key: str, + runtime_observation: RuntimeExecutionObservation, + ) -> "ExecutionStopped": + """Attribute this stop to one compiled context with the axis's records.""" + return replace( + self, + failed_combination=context_key, + runtime_observation=runtime_observation, ) + + +@dataclass(frozen=True, kw_only=True) +class ExecutionFailed(ExecutionStopped): + """A step raised while executing the axis.""" + + progress_phase = AxisErrorProgressPhase + progress_status = ErrorProgressStatus + + +@dataclass(frozen=True, kw_only=True) +class ExecutionCancelled(ExecutionStopped): + """Execution was cancelled cooperatively.""" + + stops_lane = True + progress_phase = CancelledProgressPhase + progress_status = CancelledProgressStatus diff --git a/openhcs/core/orchestrator/orchestrator.py b/openhcs/core/orchestrator/orchestrator.py index 755cc8b5bb..f315b1961e 100644 --- a/openhcs/core/orchestrator/orchestrator.py +++ b/openhcs/core/orchestrator/orchestrator.py @@ -762,7 +762,7 @@ def execute_compiled_bundle( max_workers: Optional[int] = None, visualizer: ExecutionVisualizerABC | None = None, log_file_base: Optional[str] = None, - runtime_observation_mode: RuntimeObservationMode | None = None, + runtime_observation_mode: type[RuntimeObservationMode] | None = None, debug_execution_policy: DebugExecutionPolicy = NoOpDebugExecutionPolicy(), prepared_worker_runner: "PreparedForkWorkerLaneRunner | None" = None, ) -> Dict[str, ExecutionResult]: diff --git a/openhcs/core/orchestrator/worker_execution.py b/openhcs/core/orchestrator/worker_execution.py index df2b7331b8..6b61c24822 100644 --- a/openhcs/core/orchestrator/worker_execution.py +++ b/openhcs/core/orchestrator/worker_execution.py @@ -11,7 +11,7 @@ import signal import threading from abc import ABC, abstractmethod -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import Any, Callable, Dict, Iterable, List, Mapping, Sequence from openhcs.core.compiled_execution import ( @@ -30,7 +30,11 @@ from openhcs.core.native_threading import configure_native_thread_count from openhcs.core.runtime_profile import RuntimeProfileLogger from openhcs.core.orchestrator.execution_result import ( + ExecutionCancelled, + ExecutionFailed, ExecutionResult, + ExecutionSucceeded, + ReturnAllRecords, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -71,8 +75,6 @@ AxisCompletedProgressPhase, AxisErrorProgressPhase, AxisStartedProgressPhase, - CancelledProgressPhase, - CancelledProgressStatus, ErrorProgressStatus, StartedProgressStatus, StepCompletedProgressPhase, @@ -690,7 +692,7 @@ class ExecuteInheritedLane(AnsweredWorkerCommand): lane_axis_context_keys: list[tuple[str, list[str]]] lane_context: WorkerLaneExecutionContext - runtime_observation_mode: RuntimeObservationMode + runtime_observation_mode: type[RuntimeObservationMode] def run(self, connection: Any) -> Dict[str, ExecutionResult]: execution_bundle = ForkInheritedWorkerExecutionState.require_current() @@ -1286,7 +1288,7 @@ def _execute_axis_with_sequential_combinations( pipeline_definition: List[AbstractStep], axis_contexts: TransportAxisContexts, lane_context: WorkerLaneExecutionContext, - runtime_observation_mode: RuntimeObservationMode, + runtime_observation_mode: type[RuntimeObservationMode], cancellation: ExecutionCancellationSignal | None = None, release_axis_resources: bool = True, ) -> ExecutionResult: @@ -1318,7 +1320,7 @@ def _execute_axis_with_sequential_combinations( try: cancellation.raise_if_requested(f"before context {context_key}") except ExecutionCancelledError as exc: - return ExecutionResult.cancelled( + return ExecutionCancelled( axis_id=axis_id, error_message=str(exc), runtime_observation=RuntimeExecutionObservation( @@ -1366,27 +1368,16 @@ def _execute_axis_with_sequential_combinations( lane_context.emit( axis_id=axis_id, step_name=PIPELINE_PROGRESS_STEP_NAME, - phase=( - CancelledProgressPhase - if result.is_cancelled() - else AxisErrorProgressPhase - ), - status=( - CancelledProgressStatus - if result.is_cancelled() - else ErrorProgressStatus - ), + phase=result.progress_phase, + status=result.progress_status, completed=0, total=total_steps, percent=0.0, message=result.error_message, ) - return replace( - result, - failed_combination=context_key, - runtime_observation=RuntimeExecutionObservation( - contexts=tuple(runtime_observations), - ), + return result.at_combination( + context_key, + RuntimeExecutionObservation(contexts=tuple(runtime_observations)), ) lane_context.emit( @@ -1398,7 +1389,7 @@ def _execute_axis_with_sequential_combinations( total=total_steps, percent=(completed_axis_steps / total_steps) * 100.0, ) - return ExecutionResult.success( + return ExecutionSucceeded( axis_id=axis_id, runtime_observation=RuntimeExecutionObservation( contexts=tuple(runtime_observations), @@ -1463,7 +1454,7 @@ def _execute_single_axis_static( *, context_key: str, cancellation: ExecutionCancellationSignal | None = None, - runtime_observation_mode: RuntimeObservationMode = RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode: type[RuntimeObservationMode] = ReturnAllRecords, ) -> ExecutionResult: """Execute one frozen axis context against the compiled pipeline.""" @@ -1487,7 +1478,7 @@ def _execute_single_axis_static( # original retention contract. Ordinary execution can retire dead inputs # only after the step's output/progress consumers have completed. release_consumed = ( - runtime_observation_mode is not RuntimeObservationMode.MERGE_INTO_PARENT + not runtime_observation_mode.returns_every_record and isinstance(lane_context.debug_execution_policy, NoOpDebugExecutionPolicy) and not any( plan.visualize or plan.streaming_configs @@ -1603,7 +1594,7 @@ def _execute_single_axis_static( ) except ExecutionCancelledError as exc: - return ExecutionResult.cancelled( + return ExecutionCancelled( axis_id=axis_id, error_message=str(exc), runtime_observation=RuntimeExecutionObservation.from_completed_outputs( @@ -1614,7 +1605,7 @@ def _execute_single_axis_static( logger.exception( "Axis %s failed while retaining completed output facts", axis_id ) - return ExecutionResult.error( + return ExecutionFailed( axis_id=axis_id, failed_combination=context_key, error_message=str(exc), @@ -1623,7 +1614,7 @@ def _execute_single_axis_static( ), ) - return ExecutionResult.success( + return ExecutionSucceeded( axis_id=axis_id, runtime_observation=RuntimeExecutionObservation.from_completed_outputs( {context_key: frozen_context} @@ -1635,7 +1626,7 @@ def execute_worker_lane( pipeline_definition: List[AbstractStep], lane_axis_contexts: WorkerLaneAxisContexts, lane_context: WorkerLaneExecutionContext, - runtime_observation_mode: RuntimeObservationMode, + runtime_observation_mode: type[RuntimeObservationMode], cancellation: ExecutionCancellationSignal | None = None, release_axis_resources: bool = True, ) -> Dict[str, ExecutionResult]: @@ -1654,7 +1645,7 @@ def execute_worker_lane( try: cancellation.raise_if_requested(f"before axis {axis_id}") except ExecutionCancelledError as exc: - lane_results[axis_id] = ExecutionResult.cancelled( + lane_results[axis_id] = ExecutionCancelled( axis_id=axis_id, error_message=str(exc), ) @@ -1667,6 +1658,6 @@ def execute_worker_lane( cancellation=cancellation, release_axis_resources=release_axis_resources, ) - if lane_results[axis_id].is_cancelled(): + if lane_results[axis_id].stops_lane: break return lane_results diff --git a/openhcs/core/orchestrator/worker_lanes.py b/openhcs/core/orchestrator/worker_lanes.py index be38c7686b..14196e32e7 100644 --- a/openhcs/core/orchestrator/worker_lanes.py +++ b/openhcs/core/orchestrator/worker_lanes.py @@ -118,7 +118,7 @@ class WorkerLaneExecutionPlan(ProgressExecutionContext): debug_execution_policy: DebugExecutionPolicy assignments: WorkerAssignmentPlan - runtime_observation_mode: RuntimeObservationMode + runtime_observation_mode: type[RuntimeObservationMode] def active_lane_items(self) -> tuple[tuple[str, WorkerLaneAxisContexts], ...]: """Return worker lanes that own at least one axis context.""" diff --git a/openhcs/runtime/zmq_execution_observation.py b/openhcs/runtime/zmq_execution_observation.py index 52a1d22888..04349bf56e 100644 --- a/openhcs/runtime/zmq_execution_observation.py +++ b/openhcs/runtime/zmq_execution_observation.py @@ -12,7 +12,6 @@ from openhcs.core.orchestrator.execution_result import ( ExecutionResult, RuntimeExecutionObservation, - ExecutionStatus, ) from openhcs.core.runtime_execution_validation import ( RuntimeArtifactExecutionExpectation, @@ -25,7 +24,7 @@ from openhcs.runtime.environment_provenance import RuntimeEnvironmentSnapshot ZMQ_RUNTIME_OBSERVATION_EXPORT_SCHEMA_VERSION = 8 -ZMQ_RUNTIME_OUTCOME_EXPORT_SCHEMA_VERSION = 4 +ZMQ_RUNTIME_OUTCOME_EXPORT_SCHEMA_VERSION = 5 def _axis_membership_failures( @@ -48,29 +47,12 @@ def _axis_membership_failures( return tuple(failures) -@dataclass(frozen=True, slots=True) -class ZMQExecutionAxisOutcome: - """Small per-axis result retained without worker runtime values.""" - - status: ExecutionStatus - failed_combination: str | None = None - error_message: str | None = None - - @classmethod - def from_result(cls, result: ExecutionResult) -> ZMQExecutionAxisOutcome: - return cls( - status=result.status, - failed_combination=result.failed_combination, - error_message=result.error_message, - ) - - @dataclass(frozen=True, slots=True) class ZMQRuntimeExecutionOutcomeExport: """Ordinary execution outcome without parent-side runtime-value retention.""" schema_version: int - outcomes_by_axis: Mapping[str, ZMQExecutionAxisOutcome] + outcomes_by_axis: Mapping[str, ExecutionResult] output_roots: tuple[Path, ...] compiled_axis_ids: tuple[str, ...] server_environment: RuntimeEnvironmentSnapshot | None = None @@ -91,7 +73,7 @@ def from_execution( return cls( schema_version=ZMQ_RUNTIME_OUTCOME_EXPORT_SCHEMA_VERSION, outcomes_by_axis={ - str(axis_id): ZMQExecutionAxisOutcome.from_result(result) + str(axis_id): result.without_runtime_observation() for axis_id, result in execution_results.items() }, output_roots=tuple(Path(root) for root in output_roots), @@ -130,7 +112,7 @@ def axis_count(self) -> int: @property def successful_axis_count(self) -> int: return sum( - outcome.status is ExecutionStatus.SUCCESS + outcome.is_success() for outcome in self.outcomes_by_axis.values() ) @@ -140,9 +122,9 @@ def require_successful_axes(self) -> None: self.outcomes_by_axis, ) unsuccessful = { - axis_id: outcome.status.value + axis_id: outcome.describe() for axis_id, outcome in self.outcomes_by_axis.items() - if outcome.status is not ExecutionStatus.SUCCESS + if not outcome.is_success() } failures = list(membership_failures) if unsuccessful: diff --git a/openhcs/runtime/zmq_execution_signature.py b/openhcs/runtime/zmq_execution_signature.py index 52ea245922..9e4e42a35f 100644 --- a/openhcs/runtime/zmq_execution_signature.py +++ b/openhcs/runtime/zmq_execution_signature.py @@ -68,7 +68,7 @@ def __post_init__(self) -> None: def runtime_observation_mode_for( self, execution_bundle: CompiledExecutionBundle, - ) -> RuntimeObservationMode: + ) -> type[RuntimeObservationMode]: """Resolve retention from compiled needs plus an explicit export request.""" return RuntimeObservationMode.for_compiled_bundle( diff --git a/openhcs/runtime/zmq_worker_execution.py b/openhcs/runtime/zmq_worker_execution.py index 3858572861..34d648cd24 100644 --- a/openhcs/runtime/zmq_worker_execution.py +++ b/openhcs/runtime/zmq_worker_execution.py @@ -31,7 +31,7 @@ class ZMQWorkerExecutionRequest: execution_id: str orchestrator: Any execution_bundle: CompiledExecutionBundle - runtime_observation_mode: RuntimeObservationMode + runtime_observation_mode: type[RuntimeObservationMode] progress_context: ProgressExecutionContext debug_execution_policy: Any active_execution_record: Any diff --git a/tests/integration/test_chained_callable_lineage_journey.py b/tests/integration/test_chained_callable_lineage_journey.py index 6df5605e84..a707d5e88a 100644 --- a/tests/integration/test_chained_callable_lineage_journey.py +++ b/tests/integration/test_chained_callable_lineage_journey.py @@ -23,7 +23,9 @@ PipelineConfig, ) from openhcs.core.function_patterns import MainFlowInputProjection -from openhcs.core.orchestrator.execution_result import RuntimeObservationMode +from openhcs.core.orchestrator.execution_result import ( + ReturnAllRecords, +) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec from openhcs.core.runtime_object_labels import object_label_dense_array @@ -138,7 +140,7 @@ def test_chained_public_callable_uses_declared_main_flow_not_storage_argument( results = orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="chained-public-callable-synthetic", diff --git a/tests/integration/test_image_math_publication_620.py b/tests/integration/test_image_math_publication_620.py index b849e8e233..67adef1935 100644 --- a/tests/integration/test_image_math_publication_620.py +++ b/tests/integration/test_image_math_publication_620.py @@ -17,7 +17,9 @@ GlobalPipelineConfig, LazyNapariStreamingConfig, LazyPathPlanningConfig, LazyProcessingConfig, ) -from openhcs.core.orchestrator.execution_result import RuntimeObservationMode +from openhcs.core.orchestrator.execution_result import ( + ReturnAllRecords, +) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec from openhcs.core.source_bindings import MetadataExtractionRule, MetadataSource @@ -64,7 +66,7 @@ def test_registered_mixed_image_math_publishes_once_in_both_operand_orders(tmp_p )).execution_bundle outcomes = PipelineOrchestrator(source, pipeline_config=config).initialize().execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id='engineering620-registered', diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index 72a4e105b2..b314b548d7 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -41,7 +41,9 @@ from openhcs.core.measurement_feature_queries import measurement_values_for_feature from openhcs.core.measurement_row_materialization import DataclassMeasurementColumnarRows from openhcs.core.memory import numpy -from openhcs.core.orchestrator.execution_result import RuntimeObservationMode +from openhcs.core.orchestrator.execution_result import ( + ReturnAllRecords, +) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec from openhcs.core.pipeline.function_contracts import artifact_outputs @@ -283,7 +285,7 @@ def _execute(path, document, bundle): return orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="measurement-contract-synthetic", diff --git a/tests/integration/test_primary_segmentation_diagnostics_journey.py b/tests/integration/test_primary_segmentation_diagnostics_journey.py index f94c144e61..81bec84ad1 100644 --- a/tests/integration/test_primary_segmentation_diagnostics_journey.py +++ b/tests/integration/test_primary_segmentation_diagnostics_journey.py @@ -121,7 +121,9 @@ def test_normal_compiled_runtime_persists_diagnostics_and_preserves_secondary_bi LazySourceBindingsConfig, PipelineConfig, ) - from openhcs.core.orchestrator.execution_result import RuntimeObservationMode + from openhcs.core.orchestrator.execution_result import ( + ReturnAllRecords, +) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec from openhcs.core.runtime_object_labels import object_label_dense_array @@ -294,7 +296,7 @@ def step(func, name, kwargs): results = orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="primary-diagnostics-synthetic", diff --git a/tests/integration/test_source_schema_aggregate_publication.py b/tests/integration/test_source_schema_aggregate_publication.py index 8d3d2e3f93..aeb36a2d16 100644 --- a/tests/integration/test_source_schema_aggregate_publication.py +++ b/tests/integration/test_source_schema_aggregate_publication.py @@ -29,7 +29,9 @@ PipelineConfig, ) from openhcs.core.memory import numpy as numpy_function -from openhcs.core.orchestrator.execution_result import RuntimeObservationMode +from openhcs.core.orchestrator.execution_result import ( + ReturnAllRecords, +) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline.function_contracts import artifact_outputs from openhcs.core.pipeline_document import PipelineDocumentCodec @@ -113,7 +115,7 @@ def test_real_z_aggregation_persists_scalar_and_unchanged_source_planes(tmp_path orchestrator = PipelineOrchestrator(source, pipeline_config=document.pipeline_config).initialize() outcomes = orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="source-schema-aggregate-synthetic", diff --git a/tests/unit/test_compiled_execution.py b/tests/unit/test_compiled_execution.py index 7ed7e6c760..dbc7c7f375 100644 --- a/tests/unit/test_compiled_execution.py +++ b/tests/unit/test_compiled_execution.py @@ -32,7 +32,10 @@ ) from openhcs.core.orchestrator import worker_execution from openhcs.core.orchestrator.execution_result import ( - ExecutionResult, + ExecutionSucceeded, + ReturnAllRecords, + ReturnDatasetInputRecords, + ReturnNoRecords, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -115,13 +118,13 @@ def test_compiled_execution_bundle_derives_axis_ids_from_contexts() -> None: @pytest.mark.parametrize( ("execution_scopes", "expected_mode"), ( - ((FunctionStepExecutionScope.AXIS,), RuntimeObservationMode.OMIT), + ((FunctionStepExecutionScope.AXIS,), ReturnNoRecords), ( ( FunctionStepExecutionScope.AXIS, FunctionStepExecutionScope.PLATE, ), - RuntimeObservationMode.MERGE_DATASET_INPUTS, + ReturnDatasetInputRecords, ), ), ) @@ -200,18 +203,18 @@ def test_consolidation_does_not_require_parent_payload_records( assert bundle.requires_parent_runtime_observation is False assert ( RuntimeObservationMode.for_compiled_bundle(bundle) - is RuntimeObservationMode.OMIT + is ReturnNoRecords ) def test_runtime_observation_mode_can_only_be_strengthened() -> None: assert ( - RuntimeObservationMode.OMIT.including_parent_requirement(True) - is RuntimeObservationMode.MERGE_INTO_PARENT + ReturnNoRecords.including_parent_requirement(True) + is ReturnAllRecords ) assert ( - RuntimeObservationMode.MERGE_INTO_PARENT.including_parent_requirement(False) - is RuntimeObservationMode.MERGE_INTO_PARENT + ReturnAllRecords.including_parent_requirement(False) + is ReturnAllRecords ) @@ -242,7 +245,7 @@ def execute_axis(_pipeline, context, _lane, **_kwargs): path=output_plan.path, backend="memory", ) - return ExecutionResult.success(context.axis_id) + return ExecutionSucceeded(context.axis_id) monkeypatch.setattr(worker_execution, "_execute_single_axis_static", execute_axis) monkeypatch.setattr(progress_types, "_progress_queue", SimpleNamespace(put=lambda _event: None)) @@ -259,7 +262,7 @@ def execute_axis(_pipeline, context, _lane, **_kwargs): worker_slot="worker", worker_assignments={"worker": [context.axis_id for context in contexts]}, ), - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, release_axis_resources=False, ) @@ -351,10 +354,10 @@ def execute_current_run( *, context_key, cancellation=None, - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, ): assert context_key == "A01" - assert runtime_observation_mode is RuntimeObservationMode.MERGE_INTO_PARENT + assert runtime_observation_mode is ReturnAllRecords current_records.append( context.runtime_value_store.replace( measurement_value(2), @@ -362,7 +365,7 @@ def execute_current_run( backend="memory", ) ) - return ExecutionResult.success("A01") + return ExecutionSucceeded("A01") monkeypatch.setattr( worker_execution, @@ -392,7 +395,7 @@ def execute_current_run( worker_slot="worker-0", worker_assignments={"worker-0": ["A01"]}, ), - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, ) assert result.runtime_observation.contexts == ( @@ -457,14 +460,14 @@ def fail_reset() -> None: worker_slot="worker-0", worker_assignments={"worker-0": ["A01"]}, ), - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, ) assert cleaned == [(MemoryType.CUPY, 2)] assert pixel_reference() is None -@pytest.mark.parametrize("observation_mode", tuple(RuntimeObservationMode)) +@pytest.mark.parametrize("observation_mode", tuple(RuntimeObservationMode.__subclasses__())) @pytest.mark.parametrize("release_process_resources", (True, False)) def test_worker_lane_releases_previous_axis_stack_before_next_axis( monkeypatch, @@ -483,7 +486,7 @@ def execute_axis(_pipeline, context, _lane, **_kwargs): memory_type="numpy", stack=pixels, ) - return ExecutionResult.success(context.axis_id) + return ExecutionSucceeded(context.axis_id) monkeypatch.setattr(worker_execution, "_execute_single_axis_static", execute_axis) monkeypatch.setattr(progress_types, "_progress_queue", SimpleNamespace(put=lambda _event: None)) @@ -509,7 +512,7 @@ def execute_axis(_pipeline, context, _lane, **_kwargs): @pytest.mark.parametrize( "observation_mode", - (RuntimeObservationMode.OMIT, RuntimeObservationMode.MERGE_DATASET_INPUTS), + (ReturnNoRecords, ReturnDatasetInputRecords), ) @pytest.mark.parametrize("release_process_resources", (True, False)) def test_worker_lane_releases_unconsumed_image_records_before_next_axis( @@ -534,7 +537,7 @@ def execute_axis(_pipeline, context, _lane, **_kwargs): path=output_plan.path, backend="memory", ) - return ExecutionResult.success(context.axis_id) + return ExecutionSucceeded(context.axis_id) monkeypatch.setattr(worker_execution, "_execute_single_axis_static", execute_axis) monkeypatch.setattr(progress_types, "_progress_queue", SimpleNamespace(put=lambda _event: None)) diff --git a/tests/unit/test_completed_output_publication_lifecycle.py b/tests/unit/test_completed_output_publication_lifecycle.py index dd9746b8d8..4cb02cc27f 100644 --- a/tests/unit/test_completed_output_publication_lifecycle.py +++ b/tests/unit/test_completed_output_publication_lifecycle.py @@ -16,7 +16,11 @@ from openhcs.core.debug import NoOpDebugExecutionPolicy from openhcs.core.orchestrator.cancellation import ExecutionCancellationSignal from openhcs.core.orchestrator import worker_execution -from openhcs.core.orchestrator.execution_result import RuntimeObservationMode +from openhcs.core.orchestrator.execution_result import ( + ExecutionCancelled, + ExecutionFailed, + ReturnNoRecords, +) from openhcs.core.orchestrator.worker_lanes import WorkerLaneExecutionContext from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.source_projection import OpenHCSPlaneAddress, SourcePlaneProjection @@ -106,10 +110,10 @@ def process(self, context, index): context = _context(tmp_path, 2) result = worker_execution._execute_axis_with_sequential_combinations( - [Save(), Fail()], [("A01", context)], _lane(), RuntimeObservationMode.OMIT, + [Save(), Fail()], [("A01", context)], _lane(), ReturnNoRecords, cancellation=signal, release_axis_resources=False, ) - assert result.is_cancelled() if cancel else result.is_error() + assert type(result) is (ExecutionCancelled if cancel else ExecutionFailed) assert not context.runtime_value_store.observed_values assert pixel_refs[0]() is None assert len(result.runtime_observation.contexts) == 1 diff --git a/tests/unit/test_deleted_core_names_absent.py b/tests/unit/test_deleted_core_names_absent.py index 5313b87a5c..641398d721 100644 --- a/tests/unit/test_deleted_core_names_absent.py +++ b/tests/unit/test_deleted_core_names_absent.py @@ -8,7 +8,7 @@ import openhcs from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.orchestrator.execution_result import ExecutionStatus +import openhcs.core.orchestrator.execution_result as execution_result from openhcs.core.pipeline import function_contracts OPENHCS_ROOT = Path(openhcs.__file__).parent @@ -94,5 +94,5 @@ def test_deleted_core_names_stay_absent_from_openhcs() -> None: def test_deleted_core_members_stay_absent() -> None: assert "step_type" not in {field.name for field in fields(CompiledStepPlan)} - assert "PENDING" not in ExecutionStatus.__members__ + assert not hasattr(execution_result, "ExecutionStatus") assert not hasattr(function_contracts, "validate_artifact_input_parameter_bindings") diff --git a/tests/unit/test_function_artifact_materialization.py b/tests/unit/test_function_artifact_materialization.py index 63bf6067ec..30724c0842 100644 --- a/tests/unit/test_function_artifact_materialization.py +++ b/tests/unit/test_function_artifact_materialization.py @@ -66,9 +66,10 @@ from openhcs.core.post_execute import PostExecuteHook from openhcs.domains.microscopy.analysis_consolidation import AnalysisConsolidationHook from openhcs.core.orchestrator.execution_result import ( + ReturnAllRecords, + ReturnNoRecords, RuntimeContextObservation, RuntimeExecutionObservation, - RuntimeObservationMode, ) from openhcs.core.pipeline.function_contracts import artifact_outputs from openhcs.core.pipeline.artifact_planning import ( @@ -2006,7 +2007,7 @@ def test_consolidation_preserves_export_bundle_and_text_tables(options, payload) @pytest.mark.parametrize( "observation_mode", - (RuntimeObservationMode.OMIT, RuntimeObservationMode.MERGE_INTO_PARENT), + (ReturnNoRecords, ReturnAllRecords), ) def test_completed_observation_projects_tables_before_worker_payload_release( observation_mode, monkeypatch, @@ -2068,10 +2069,10 @@ def test_completed_observation_projects_tables_before_worker_payload_release( context.runtime_value_store.clear() del pixels assert (pixel_reference() is not None) is ( - observation_mode is RuntimeObservationMode.MERGE_INTO_PARENT + observation_mode is ReturnAllRecords ) assert len(observation.records) == ( - 2 if observation_mode is RuntimeObservationMode.MERGE_INTO_PARENT else 0 + 2 if observation_mode is ReturnAllRecords else 0 ) assert len(observation.outputs.runtime_export_paths) == 1 assert observation.outputs.runtime_export_paths[0].suffix == ".csv" diff --git a/tests/unit/test_function_step_execution_scope.py b/tests/unit/test_function_step_execution_scope.py index b14444ece9..27fb88c049 100644 --- a/tests/unit/test_function_step_execution_scope.py +++ b/tests/unit/test_function_step_execution_scope.py @@ -45,7 +45,11 @@ execute_dataset_scoped_steps, validate_dataset_scoped_contexts, ) -from openhcs.core.orchestrator.execution_result import RuntimeObservationMode +from openhcs.core.orchestrator.execution_result import ( + ReturnAllRecords, + ReturnDatasetInputRecords, + ReturnNoRecords, +) from openhcs.core.progress import ProgressEvent, ProgressExecutionContext from openhcs.core.pipeline.function_contracts import ( artifact_inputs, @@ -684,7 +688,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): count=3, ) observations = context.runtime_value_store.observed_values - retained = RuntimeObservationMode.MERGE_DATASET_INPUTS.retain_records( + retained = ReturnDatasetInputRecords.retain_records( observations, context, ) @@ -692,13 +696,13 @@ def export(*, artifact_batch: RuntimeArtifactBatch): (["raw"] if stored_image else []) + ["Measurements", "Measurements"] ) assert ( - RuntimeObservationMode.MERGE_INTO_PARENT.retain_records( + ReturnAllRecords.retain_records( observations, context, ) is observations ) - assert RuntimeObservationMode.OMIT.retain_records(observations, context) == () + assert ReturnNoRecords.retain_records(observations, context) == () full_store = type(context.runtime_value_store)() full_store.merge_observed_values(observations) @@ -752,7 +756,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): metadata_writer=True, ) context = _plate_context("A01", (plan,)) - assert RuntimeObservationMode.MERGE_DATASET_INPUTS.retain_records((), context) == () + assert ReturnDatasetInputRecords.retain_records((), context) == () with pytest.raises(ValueError, match="missing required input"): _dataset_artifact_batch( compiled_contexts={"A01": context}, diff --git a/tests/unit/test_materialization_batch_outcomes.py b/tests/unit/test_materialization_batch_outcomes.py index 81777cea88..969639b7e7 100644 --- a/tests/unit/test_materialization_batch_outcomes.py +++ b/tests/unit/test_materialization_batch_outcomes.py @@ -22,7 +22,7 @@ BundleExecutionResults, ) from openhcs.core.orchestrator.execution_result import ( - ExecutionResult, + ExecutionSucceeded, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -163,7 +163,7 @@ def release_execution_image_cache(self): self.released = True -@pytest.mark.parametrize("mode", tuple(RuntimeObservationMode)) +@pytest.mark.parametrize("mode", tuple(RuntimeObservationMode.__subclasses__())) def test_worker_exports_actual_step_outcomes_after_resource_release( tmp_path, monkeypatch, mode ): @@ -219,7 +219,7 @@ def test_parent_and_worker_exports_share_existing_observation_authority(tmp_path contexts=(RuntimeContextObservation("A01", (), StepExecutionObservation({}, (plate_path,))),) ) results = BundleExecutionResults( - {"A01": ExecutionResult.success("A01", runtime_observation=worker)}, + {"A01": ExecutionSucceeded("A01", runtime_observation=worker)}, extras=BundleExecutionExtras( viewer_states_by_port={}, runtime_observation=parent ), diff --git a/tests/unit/test_nonpersistent_viewer_state_evidence.py b/tests/unit/test_nonpersistent_viewer_state_evidence.py index 55f042a152..d715ad54b5 100644 --- a/tests/unit/test_nonpersistent_viewer_state_evidence.py +++ b/tests/unit/test_nonpersistent_viewer_state_evidence.py @@ -14,8 +14,8 @@ BundleExecutionResults, ) from openhcs.core.orchestrator.execution_result import ( - ExecutionResult, - RuntimeObservationMode, + ExecutionSucceeded, + ReturnNoRecords, ) from openhcs.runtime.viewer_protocol import ViewerControlResponse from openhcs.runtime.zmq_server_hooks import ZMQResultsSummaryEnricher @@ -47,7 +47,7 @@ def test_settled_viewer_state_survives_zmq_execution_result_transport( } ) compiled_results = BundleExecutionResults( - {"A01": ExecutionResult.success("A01")}, + {"A01": ExecutionSucceeded("A01")}, extras=BundleExecutionExtras(viewer_states_by_port={5563: response}), ) @@ -90,7 +90,7 @@ def execute_compiled_bundle(self, **kwargs): execution_id="exec", orchestrator=Orchestrator(), execution_bundle=execution_bundle, - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, progress_context=ProgressExecutionContext( execution_id="exec", subject_id="plate", @@ -101,7 +101,7 @@ def execute_compiled_bundle(self, **kwargs): ).execute() assert returned is compiled_results - assert execution_calls[0]["runtime_observation_mode"] is RuntimeObservationMode.OMIT + assert execution_calls[0]["runtime_observation_mode"] is ReturnNoRecords assert progress_queue.values == [None] assert record.get_extra(BundleExecutionExtras.EXECUTION_RECORD_KEY) is ( compiled_results.extras diff --git a/tests/unit/test_openhcs_measured_run.py b/tests/unit/test_openhcs_measured_run.py index b5f47b386c..825a8b922a 100644 --- a/tests/unit/test_openhcs_measured_run.py +++ b/tests/unit/test_openhcs_measured_run.py @@ -20,7 +20,9 @@ from benchmark.contracts.tool_adapter import ToolExecutionError from benchmark.timing import BenchmarkPhase, PhaseTimingTrace from openhcs.core.config import GlobalPipelineConfig, PipelineConfig -from openhcs.core.orchestrator.execution_result import ExecutionResult +from openhcs.core.orchestrator.execution_result import ( + ExecutionSucceeded, +) from openhcs.core.pipeline_document import PipelineDocumentCodec from openhcs.runtime.zmq_application import OPENHCS_ENDPOINT_APPLICATION from openhcs.runtime.zmq_execution_client import OpenHCSExecutionSubmission, ZMQExecutionRequestBuilder @@ -467,7 +469,7 @@ def test_outcome_only_run_uses_the_shared_receipt_finalizer(tmp_path: Path) -> N observation_path = tmp_path / "outcomes.pkl.gz" ZMQRuntimeExecutionOutcomeExport.from_execution( compiled_axis_ids=("A01",), - execution_results={"A01": ExecutionResult.success("A01")}, + execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path / "output",), execution_id="execution-1", ).write(observation_path) @@ -541,7 +543,7 @@ def test_outcome_only_run_uses_the_shared_receipt_finalizer(tmp_path: Path) -> N observation_path.unlink() ZMQRuntimeExecutionOutcomeExport.from_execution( compiled_axis_ids=("A01",), - execution_results={"A01": ExecutionResult.success("A01")}, + execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path / "output",), execution_id="execution-1", ).write(observation_path) diff --git a/tests/unit/test_orchestrator_execution_result.py b/tests/unit/test_orchestrator_execution_result.py index 0d59f20d91..54bda1f147 100644 --- a/tests/unit/test_orchestrator_execution_result.py +++ b/tests/unit/test_orchestrator_execution_result.py @@ -9,11 +9,14 @@ BundleExecutionResults, ) -from openhcs.core.orchestrator.execution_result import ExecutionResult +from openhcs.core.orchestrator.execution_result import ( + ExecutionFailed, + ExecutionSucceeded, +) def test_execution_result_transport_preserves_mapping_proxy_metadata() -> None: - result = ExecutionResult.success(axis_id="W001") + result = ExecutionSucceeded(axis_id="W001") payload = {"result": result, "metadata": MappingProxyType({"well": "W001"})} restored = pickle.loads(pickle.dumps(payload)) @@ -25,16 +28,16 @@ def test_execution_result_transport_preserves_mapping_proxy_metadata() -> None: @pytest.mark.parametrize("failed", [False, True]) def test_plate_terminal_outcome_preserves_individual_axes(failed: bool) -> None: - successful = ExecutionResult.success("A01") + successful = ExecutionSucceeded("A01") other = ( - ExecutionResult.error("A02", error_message="producer lineage missing") + ExecutionFailed("A02", error_message="producer lineage missing") if failed - else ExecutionResult.success("A02") + else ExecutionSucceeded("A02") ) results = BundleExecutionResults({"A01": successful, "A02": other}) assert results.is_success() is not failed if failed: - with pytest.raises(RuntimeError, match="A02: error: producer lineage missing"): + with pytest.raises(RuntimeError, match="A02: ExecutionFailed: producer lineage missing"): results.require_success() else: results.require_success() diff --git a/tests/unit/test_orchestrator_lane_planning.py b/tests/unit/test_orchestrator_lane_planning.py index e4b2d24ec4..c80ae8cf07 100644 --- a/tests/unit/test_orchestrator_lane_planning.py +++ b/tests/unit/test_orchestrator_lane_planning.py @@ -45,9 +45,12 @@ wait_until_visualizers_ready, ) from openhcs.core.orchestrator.execution_result import ( - ExecutionResult, + ExecutionCancelled, + ExecutionFailed, + ExecutionSucceeded, + ReturnAllRecords, + ReturnNoRecords, RuntimeExecutionObservation, - RuntimeObservationMode, ) from openhcs.core.orchestrator.worker_execution import ( ForkInheritedWorkerExecutorResources, @@ -99,7 +102,7 @@ def test_fork_resources_reuse_lane_processes_for_ordered_plate_partitions(monkey worker_execution_module, "execute_worker_lane", lambda *, lane_axis_contexts, **_kwargs: { - axis: ExecutionResult.success(axis) for axis, _ in lane_axis_contexts + axis: ExecutionSucceeded(axis) for axis, _ in lane_axis_contexts }, ) ForkInheritedWorkerExecutionState.install( @@ -120,7 +123,7 @@ def test_fork_resources_reuse_lane_processes_for_ordered_plate_partitions(monkey worker_assignments={"worker_0": ["A01"], "worker_1": ["A02"]}, lane_axis_contexts={"worker_0": [("A01", [])], "worker_1": [("A02", [])]}, ), - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, ) processes = [] try: @@ -156,7 +159,7 @@ def test_fork_resources_join_all_lanes_after_plate_failure(monkeypatch, failure) worker_execution_module, "execute_worker_lane", lambda *, lane_axis_contexts, **_kwargs: { - axis: ExecutionResult.success(axis) for axis, _ in lane_axis_contexts + axis: ExecutionSucceeded(axis) for axis, _ in lane_axis_contexts }, ) ForkInheritedWorkerExecutionState.install( @@ -177,7 +180,7 @@ def test_fork_resources_join_all_lanes_after_plate_failure(monkeypatch, failure) worker_assignments={"worker_0": ["A01"], "worker_1": ["A02"]}, lane_axis_contexts={"worker_0": [("A01", [])], "worker_1": [("A02", [])]}, ), - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, ) processes = [] try: @@ -250,7 +253,7 @@ def _execute_with_visualizer(monkeypatch, visualizer, *, progress_queue=None): visualizer=None, log_file_base=None, progress_queue=progress_queue or SimpleNamespace(put=lambda _event: None), - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, debug_execution_policy=NoOpDebugExecutionPolicy(), ) @@ -270,7 +273,7 @@ def plan_worker_lanes(self, **_kwargs): ) def run_worker_lanes(self, **_kwargs): - return {"A01": ExecutionResult.success("A01")} + return {"A01": ExecutionSucceeded("A01")} def shutdown_executor(self): return None @@ -387,7 +390,7 @@ def test_compiled_plate_execution_request_uses_bundle_as_runtime_authority(): visualizer=None, log_file_base=None, progress_queue="queue", - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, debug_execution_policy=NoOpDebugExecutionPolicy(), ) orchestrator = SimpleNamespace( @@ -424,7 +427,7 @@ def test_plate_scope_rejects_omitted_runtime_observations(monkeypatch): visualizer=None, log_file_base=None, progress_queue="queue", - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, debug_execution_policy=NoOpDebugExecutionPolicy(), ) monkeypatch.setattr( @@ -433,7 +436,7 @@ def test_plate_scope_rejects_omitted_runtime_observations(monkeypatch): lambda _contexts: (1,), ) - with pytest.raises(ValueError, match="MERGE_INTO_PARENT"): + with pytest.raises(ValueError, match="runtime observation mode that returns records"): validate_bundle_execution( SimpleNamespace(is_initialized=lambda: True), request, @@ -767,7 +770,7 @@ def held_lane(**_kwargs): writer, (reader,), worker_execution_module.ExecuteInheritedLane( - [], lane, RuntimeObservationMode.OMIT + [], lane, ReturnNoRecords ), ), ) @@ -816,7 +819,7 @@ def fake_execute_worker_lane( ): assert release_axis_resources is True axis_id = lane_context.owned_partition_values[0] - return {axis_id: ExecutionResult.success(axis_id)} + return {axis_id: ExecutionSucceeded(axis_id)} monkeypatch.setattr( worker_execution_module.concurrent.futures, @@ -837,7 +840,7 @@ def fake_execute_worker_lane( worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", [("A01", object())])]}, ), - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, ) results = PooledWorkerLaneRunner(FakeExecutor(), cancellation=None).run( @@ -846,7 +849,7 @@ def fake_execute_worker_lane( parent_contexts={}, ) - assert results == {"A01": ExecutionResult.success("A01")} + assert results == {"A01": ExecutionSucceeded("A01")} def test_worker_lane_honours_cancellation_before_next_axis(monkeypatch): @@ -865,7 +868,7 @@ def execute_axis( axis_id = axis_contexts[0][1].axis_id visited.append(axis_id) cancellation.request() - return ExecutionResult.success(axis_id) + return ExecutionSucceeded(axis_id) monkeypatch.setattr( worker_execution_module, @@ -888,13 +891,13 @@ def execute_axis( pipeline_definition=[], lane_axis_contexts=lane_axis_contexts, lane_context=lane_context, - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, cancellation=cancellation, ) assert visited == ["A01"] assert results["A01"].is_success() - assert results["B01"].is_cancelled() + assert type(results["B01"]) is ExecutionCancelled assert "before axis B01" in results["B01"].error_message @@ -941,7 +944,7 @@ def fake_execute_worker_lane( ): assert release_axis_resources is True submitted_pipeline.extend(pipeline_definition) - return {"A01": ExecutionResult.success("A01")} + return {"A01": ExecutionSucceeded("A01")} monkeypatch.setattr( worker_execution_module.concurrent.futures, @@ -962,7 +965,7 @@ def fake_execute_worker_lane( worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", [("A01", object())])]}, ), - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, ) PooledWorkerLaneRunner(FakeExecutor(), cancellation=None).run( @@ -998,7 +1001,7 @@ def fake_execute_worker_lane( ): executed.append((pipeline_definition, lane_axis_contexts)) return { - "A01": ExecutionResult.success( + "A01": ExecutionSucceeded( "A01", runtime_observation=MergeForbiddenObservation(), ) @@ -1030,7 +1033,7 @@ def fake_execute_worker_lane( worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", ["A01"])]}, ), - runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + runtime_observation_mode=ReturnAllRecords, ) results = ForkInheritedWorkerLaneRunner(ForbiddenMultiprocessingContext()).run( @@ -1071,7 +1074,7 @@ def submit(self, fn, *args): worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", [("A01", object())])]}, ), - runtime_observation_mode=RuntimeObservationMode.OMIT, + runtime_observation_mode=ReturnNoRecords, ) with pytest.raises(RuntimeError, match="lane exploded"): @@ -1137,13 +1140,13 @@ def test_execution_state_projector_maps_success_and_failure(): project_execution_state( orchestrator, - BundleExecutionResults({"A01": ExecutionResult.success("A01")}), + BundleExecutionResults({"A01": ExecutionSucceeded("A01")}), ) assert orchestrator._state is orchestrator_module.OrchestratorState.COMPLETED project_execution_state( orchestrator, - BundleExecutionResults({"A01": ExecutionResult.error("A01")}), + BundleExecutionResults({"A01": ExecutionFailed("A01", error_message="failed")}), ) assert orchestrator._state is orchestrator_module.OrchestratorState.EXEC_FAILED @@ -1393,7 +1396,7 @@ def force_stop(self): ) assert isinstance(results, BundleExecutionResults) - assert results == {"A01": ExecutionResult.success("A01")} + assert results == {"A01": ExecutionSucceeded("A01")} assert results.extras.viewer_states_by_port == {5563: response} layer = results.extras.viewer_states_by_port[5563].payload["layers"][0] assert layer["route_key"] == "step-2:objects" diff --git a/tests/unit/test_runtime_execution_validation.py b/tests/unit/test_runtime_execution_validation.py index 42d3048c6f..66f20f7a9f 100644 --- a/tests/unit/test_runtime_execution_validation.py +++ b/tests/unit/test_runtime_execution_validation.py @@ -29,7 +29,7 @@ MeasurementSparseColumnarRows, ) from openhcs.core.orchestrator.execution_result import ( - ExecutionResult, + ExecutionSucceeded, RuntimeContextObservation, RuntimeExecutionObservation, ) @@ -234,7 +234,7 @@ def test_zmq_observation_exports_exact_compiler_owned_artifacts( export = ZMQRuntimeExecutionObservationExport.from_execution( compiled_contexts={"A01": context}, - execution_results={"A01": ExecutionResult.success("A01")}, + execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path,), runtime_observations=runtime_observations, ) @@ -561,7 +561,7 @@ def test_zmq_observation_compresses_and_preserves_exact_runtime_records( ) export = ZMQRuntimeExecutionObservationExport.from_execution( compiled_contexts={"A01": context}, - execution_results={"A01": ExecutionResult.success("A01")}, + execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path,), runtime_observations=(), ) diff --git a/tests/unit/test_well_throughput_scaling.py b/tests/unit/test_well_throughput_scaling.py index 644cf7b1c8..239457daf4 100644 --- a/tests/unit/test_well_throughput_scaling.py +++ b/tests/unit/test_well_throughput_scaling.py @@ -47,7 +47,9 @@ PipelineConfig, PartitionFilterConfig, ) -from openhcs.core.orchestrator.execution_result import ExecutionResult +from openhcs.core.orchestrator.execution_result import ( + ExecutionSucceeded, +) from openhcs.core.source_projection import ( OpenHCSPlaneAddress, SourcePlaneProjection, @@ -1398,7 +1400,7 @@ def fake_execute( f"W{index:03d}" for index in range(1, well_count + 1) ), execution_results={ - f"W{index:03d}": ExecutionResult.success(f"W{index:03d}") + f"W{index:03d}": ExecutionSucceeded(f"W{index:03d}") for index in range(1, well_count + 1) }, output_roots=(tmp_path / "output",), diff --git a/tests/unit/test_zmq_execution_server.py b/tests/unit/test_zmq_execution_server.py index 2f58dbb20d..863bd343de 100644 --- a/tests/unit/test_zmq_execution_server.py +++ b/tests/unit/test_zmq_execution_server.py @@ -27,10 +27,13 @@ BundleExecutionResults, ) from openhcs.core.orchestrator.execution_result import ( - ExecutionResult, + ExecutionFailed, + ExecutionSucceeded, + ReturnAllRecords, + ReturnDatasetInputRecords, + ReturnNoRecords, RuntimeContextObservation, RuntimeExecutionObservation, - RuntimeObservationMode, ) from openhcs.core.progress import ProgressEvent, ProgressExecutionContext from openhcs.runtime.zmq_execution_observation import ( @@ -108,11 +111,11 @@ def test_axis_outcomes_qualify_server_terminal_after_export(monkeypatch, failed) server._lifecycle.enqueue(record) results = BundleExecutionResults( { - "A01": ExecutionResult.success("A01"), + "A01": ExecutionSucceeded("A01"), "A02": ( - ExecutionResult.error("A02", error_message="producer lineage missing") + ExecutionFailed("A02", error_message="producer lineage missing") if failed - else ExecutionResult.success("A02") + else ExecutionSucceeded("A02") ), } ) @@ -131,7 +134,7 @@ def test_axis_outcomes_qualify_server_terminal_after_export(monkeypatch, failed) compile_only=False, execution_id=record.execution_id, auxiliary_params=SimpleNamespace( - runtime_observation_mode_for=lambda bundle: RuntimeObservationMode.OMIT + runtime_observation_mode_for=lambda bundle: ReturnNoRecords ), ) monkeypatch.setattr( @@ -158,7 +161,7 @@ def test_axis_outcomes_qualify_server_terminal_after_export(monkeypatch, failed) assert exported == [results] assert results["A01"].is_success() if failed: - assert "A02: error: producer lineage missing" in terminal[MessageFields.ERROR] + assert "A02: ExecutionFailed: producer lineage missing" in terminal[MessageFields.ERROR] else: assert terminal[MessageFields.RESULTS_SUMMARY][MessageFields.PARTITION_VALUES] == [ "A01", @@ -222,26 +225,26 @@ def test_terminal_notification_contains_openhcs_summary_before_cleanup( @pytest.mark.parametrize( ("compiled_mode", "export_path", "expected_mode"), ( - (RuntimeObservationMode.OMIT, None, RuntimeObservationMode.OMIT), + (ReturnNoRecords, None, ReturnNoRecords), ( - RuntimeObservationMode.OMIT, + ReturnNoRecords, "/tmp/runtime-observation.pkl", - RuntimeObservationMode.MERGE_INTO_PARENT, + ReturnAllRecords, ), ( - RuntimeObservationMode.MERGE_DATASET_INPUTS, + ReturnDatasetInputRecords, None, - RuntimeObservationMode.MERGE_DATASET_INPUTS, + ReturnDatasetInputRecords, ), ( - RuntimeObservationMode.MERGE_DATASET_INPUTS, + ReturnDatasetInputRecords, "/tmp/runtime-observation.pkl", - RuntimeObservationMode.MERGE_INTO_PARENT, + ReturnAllRecords, ), ( - RuntimeObservationMode.MERGE_INTO_PARENT, + ReturnAllRecords, None, - RuntimeObservationMode.MERGE_DATASET_INPUTS, + ReturnDatasetInputRecords, ), ), ) @@ -273,7 +276,7 @@ def test_outcome_export_does_not_strengthen_worker_runtime_value_retention() -> assert ( params.runtime_observation_mode_for(execution_bundle) - is RuntimeObservationMode.OMIT + is ReturnNoRecords ) assert ZMQAuxiliaryExecutionParams.from_transport(params.to_transport()) == params @@ -325,7 +328,7 @@ def test_server_exports_outcomes_without_projecting_compiled_values( compilation=compilation, execution_results=BundleExecutionResults( { - "A01": ExecutionResult.success( + "A01": ExecutionSucceeded( "A01", runtime_observation=RuntimeExecutionObservation( contexts=( diff --git a/tests/unit/test_zmq_runtime_outcome_export.py b/tests/unit/test_zmq_runtime_outcome_export.py index 3f624d105e..7b9faf19e7 100644 --- a/tests/unit/test_zmq_runtime_outcome_export.py +++ b/tests/unit/test_zmq_runtime_outcome_export.py @@ -5,8 +5,8 @@ import pytest from openhcs.core.orchestrator.execution_result import ( - ExecutionResult, - ExecutionStatus, + ExecutionFailed, + ExecutionSucceeded, RuntimeContextObservation, RuntimeExecutionObservation, ) @@ -25,7 +25,7 @@ def test_outcome_export_round_trip_excludes_runtime_values(tmp_path: Path) -> No declared_output.write_bytes(b"sqlite evidence") owned_exports = RuntimeExportObservation.from_output_paths((declared_output,)) execution_results = { - "A01": ExecutionResult.success( + "A01": ExecutionSucceeded( "A01", runtime_observation=RuntimeExecutionObservation( contexts=( @@ -37,7 +37,7 @@ def test_outcome_export_round_trip_excludes_runtime_values(tmp_path: Path) -> No ) ), ), - "B01": ExecutionResult.error( + "B01": ExecutionFailed( "B01", failed_combination="field-2", error_message="failed" ), } @@ -61,12 +61,12 @@ def test_outcome_export_round_trip_excludes_runtime_values(tmp_path: Path) -> No assert restored.execution_id == "execution-1" assert restored.axis_count == 2 assert restored.successful_axis_count == 1 - assert restored.outcomes_by_axis["B01"].status is ExecutionStatus.ERROR + assert type(restored.outcomes_by_axis["B01"]) is ExecutionFailed assert restored.outcomes_by_axis["B01"].failed_combination == "field-2" assert restored.exports is not None assert restored.exports.output_files == (declared_output,) assert runtime_value not in path.read_bytes() - with pytest.raises(RuntimeError, match="'B01': 'error'"): + with pytest.raises(RuntimeError, match="'B01': 'ExecutionFailed: failed'"): restored.require_successful_axes() with pytest.raises( TypeError, match="must contain ZMQRuntimeExecutionObservationExport" @@ -77,7 +77,7 @@ def test_outcome_export_round_trip_excludes_runtime_values(tmp_path: Path) -> No def test_successful_outcome_export_accepts_all_axes(tmp_path: Path) -> None: exported = ZMQRuntimeExecutionOutcomeExport.from_execution( compiled_axis_ids=("A01",), - execution_results={"A01": ExecutionResult.success("A01")}, + execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path,), ) @@ -87,7 +87,7 @@ def test_successful_outcome_export_accepts_all_axes(tmp_path: Path) -> None: def test_outcome_export_rejects_missing_and_uncompiled_axes(tmp_path: Path) -> None: missing = ZMQRuntimeExecutionOutcomeExport.from_execution( compiled_axis_ids=("A01", "B01"), - execution_results={"A01": ExecutionResult.success("A01")}, + execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path,), ) with pytest.raises(RuntimeError, match="compiled axes have no execution outcome"): @@ -96,8 +96,8 @@ def test_outcome_export_rejects_missing_and_uncompiled_axes(tmp_path: Path) -> N unexpected = ZMQRuntimeExecutionOutcomeExport.from_execution( compiled_axis_ids=("A01",), execution_results={ - "A01": ExecutionResult.success("A01"), - "B01": ExecutionResult.success("B01"), + "A01": ExecutionSucceeded("A01"), + "B01": ExecutionSucceeded("B01"), }, output_roots=(tmp_path,), ) @@ -108,7 +108,7 @@ def test_outcome_export_rejects_missing_and_uncompiled_axes(tmp_path: Path) -> N def test_value_export_rejects_an_uncompiled_execution_axis() -> None: exported = ZMQRuntimeExecutionObservationExport.from_execution( compiled_contexts={}, - execution_results={"A01": ExecutionResult.success("A01")}, + execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(), runtime_observations=(), ) From aab9e55169c3580144566628f3e9c44c424dd325 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 14:56:39 -0400 Subject: [PATCH 03/13] WIP: K7 observation modes take E1's names Co-Authored-By: Claude Opus 5.5 --- .../K7-execution-protocol.md | 2 +- openhcs/core/orchestrator/bundle_execution.py | 4 +-- openhcs/core/orchestrator/execution_result.py | 12 +++---- openhcs/core/orchestrator/worker_execution.py | 4 +-- .../test_chained_callable_lineage_journey.py | 4 +-- .../test_image_math_publication_620.py | 4 +-- .../test_measurement_declaration_journey.py | 4 +-- ...rimary_segmentation_diagnostics_journey.py | 4 +-- ...est_source_schema_aggregate_publication.py | 4 +-- tests/unit/test_compiled_execution.py | 32 +++++++++---------- ..._completed_output_publication_lifecycle.py | 4 +-- .../test_function_artifact_materialization.py | 10 +++--- .../test_function_step_execution_scope.py | 14 ++++---- ...est_nonpersistent_viewer_state_evidence.py | 6 ++-- tests/unit/test_orchestrator_lane_planning.py | 26 +++++++-------- tests/unit/test_zmq_execution_server.py | 28 ++++++++-------- 16 files changed, 81 insertions(+), 81 deletions(-) diff --git a/docs/refactor/consolidation-20261009/K7-execution-protocol.md b/docs/refactor/consolidation-20261009/K7-execution-protocol.md index 29022ee775..7a460731d1 100644 --- a/docs/refactor/consolidation-20261009/K7-execution-protocol.md +++ b/docs/refactor/consolidation-20261009/K7-execution-protocol.md @@ -47,7 +47,7 @@ The guards pass; touched tests pass; official30 is 30/30; the structural ratchet ## Progress - [x] Receipt. -- [x] ExecutionResult and RuntimeObservationMode families (`ExecutionSucceeded`/`ExecutionFailed`/`ExecutionCancelled`; `ReturnNoRecords`/`ReturnAllRecords`/`ReturnDatasetInputRecords`; `ZMQExecutionAxisOutcome` deleted, outcome export stores results without records). +- [x] ExecutionResult and RuntimeObservationMode families (`ExecutionSucceeded`/`ExecutionFailed`/`ExecutionCancelled`; `OmitRuntimeRecords`/`MergeAllRuntimeRecords`/`MergeDatasetInputRecords`; `ZMQExecutionAxisOutcome` deleted, outcome export stores results without records). - [ ] ComponentGroupScope variants, plan `group_scope` field, compose split. - [ ] Context phase types. - [ ] Guards, tests, ratchet, parity, PR. diff --git a/openhcs/core/orchestrator/bundle_execution.py b/openhcs/core/orchestrator/bundle_execution.py index 48c43136f9..9c43e0976f 100644 --- a/openhcs/core/orchestrator/bundle_execution.py +++ b/openhcs/core/orchestrator/bundle_execution.py @@ -41,7 +41,7 @@ from openhcs.core.orchestrator.cancellation import ExecutionCancelledError from openhcs.core.orchestrator.execution_result import ( ExecutionResult, - ReturnAllRecords, + MergeAllRuntimeRecords, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -655,7 +655,7 @@ def execute_dataset_scoped_steps( context_key=context_key, context=context, records=records, - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, outputs=StepExecutionObservation.combine( observations_by_context[context_key] ), diff --git a/openhcs/core/orchestrator/execution_result.py b/openhcs/core/orchestrator/execution_result.py index 67205c4830..585bd3eb31 100644 --- a/openhcs/core/orchestrator/execution_result.py +++ b/openhcs/core/orchestrator/execution_result.py @@ -123,8 +123,8 @@ def for_compiled_bundle( """Consolidation carries rendered tables; only dataset-scoped inputs need records.""" if bundle.requires_parent_runtime_observation: - return ReturnDatasetInputRecords - return ReturnNoRecords + return MergeDatasetInputRecords + return OmitRuntimeRecords @classmethod def including_parent_requirement( @@ -133,7 +133,7 @@ def including_parent_requirement( ) -> type["RuntimeObservationMode"]: """Return this mode strengthened by an additional retention requirement.""" - return ReturnAllRecords if required else cls + return MergeAllRuntimeRecords if required else cls @classmethod @abstractmethod @@ -145,7 +145,7 @@ def retain_records( """Keep records consumed by the selected parent-side execution route.""" -class ReturnNoRecords(RuntimeObservationMode): +class OmitRuntimeRecords(RuntimeObservationMode): """Return no runtime records; the parent needs none.""" collects_records = False @@ -155,7 +155,7 @@ def retain_records(cls, records, context): return () -class ReturnAllRecords(RuntimeObservationMode): +class MergeAllRuntimeRecords(RuntimeObservationMode): """Return every runtime record to the parent.""" returns_every_record = True @@ -165,7 +165,7 @@ def retain_records(cls, records, context): return records -class ReturnDatasetInputRecords(RuntimeObservationMode): +class MergeDatasetInputRecords(RuntimeObservationMode): """Return only the records dataset-scoped steps consume in the parent.""" @classmethod diff --git a/openhcs/core/orchestrator/worker_execution.py b/openhcs/core/orchestrator/worker_execution.py index 6b61c24822..0ca3555ccb 100644 --- a/openhcs/core/orchestrator/worker_execution.py +++ b/openhcs/core/orchestrator/worker_execution.py @@ -34,7 +34,7 @@ ExecutionFailed, ExecutionResult, ExecutionSucceeded, - ReturnAllRecords, + MergeAllRuntimeRecords, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -1454,7 +1454,7 @@ def _execute_single_axis_static( *, context_key: str, cancellation: ExecutionCancellationSignal | None = None, - runtime_observation_mode: type[RuntimeObservationMode] = ReturnAllRecords, + runtime_observation_mode: type[RuntimeObservationMode] = MergeAllRuntimeRecords, ) -> ExecutionResult: """Execute one frozen axis context against the compiled pipeline.""" diff --git a/tests/integration/test_chained_callable_lineage_journey.py b/tests/integration/test_chained_callable_lineage_journey.py index a707d5e88a..9abae056aa 100644 --- a/tests/integration/test_chained_callable_lineage_journey.py +++ b/tests/integration/test_chained_callable_lineage_journey.py @@ -24,7 +24,7 @@ ) from openhcs.core.function_patterns import MainFlowInputProjection from openhcs.core.orchestrator.execution_result import ( - ReturnAllRecords, + MergeAllRuntimeRecords, ) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec @@ -140,7 +140,7 @@ def test_chained_public_callable_uses_declared_main_flow_not_storage_argument( results = orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="chained-public-callable-synthetic", diff --git a/tests/integration/test_image_math_publication_620.py b/tests/integration/test_image_math_publication_620.py index 67adef1935..52f0ac7e82 100644 --- a/tests/integration/test_image_math_publication_620.py +++ b/tests/integration/test_image_math_publication_620.py @@ -18,7 +18,7 @@ LazyProcessingConfig, ) from openhcs.core.orchestrator.execution_result import ( - ReturnAllRecords, + MergeAllRuntimeRecords, ) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec @@ -66,7 +66,7 @@ def test_registered_mixed_image_math_publishes_once_in_both_operand_orders(tmp_p )).execution_bundle outcomes = PipelineOrchestrator(source, pipeline_config=config).initialize().execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id='engineering620-registered', diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index b314b548d7..40529b06a0 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -42,7 +42,7 @@ from openhcs.core.measurement_row_materialization import DataclassMeasurementColumnarRows from openhcs.core.memory import numpy from openhcs.core.orchestrator.execution_result import ( - ReturnAllRecords, + MergeAllRuntimeRecords, ) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec @@ -285,7 +285,7 @@ def _execute(path, document, bundle): return orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="measurement-contract-synthetic", diff --git a/tests/integration/test_primary_segmentation_diagnostics_journey.py b/tests/integration/test_primary_segmentation_diagnostics_journey.py index 81bec84ad1..9ff0e70d64 100644 --- a/tests/integration/test_primary_segmentation_diagnostics_journey.py +++ b/tests/integration/test_primary_segmentation_diagnostics_journey.py @@ -122,7 +122,7 @@ def test_normal_compiled_runtime_persists_diagnostics_and_preserves_secondary_bi PipelineConfig, ) from openhcs.core.orchestrator.execution_result import ( - ReturnAllRecords, + MergeAllRuntimeRecords, ) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline_document import PipelineDocumentCodec @@ -296,7 +296,7 @@ def step(func, name, kwargs): results = orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="primary-diagnostics-synthetic", diff --git a/tests/integration/test_source_schema_aggregate_publication.py b/tests/integration/test_source_schema_aggregate_publication.py index aeb36a2d16..30a9fc3941 100644 --- a/tests/integration/test_source_schema_aggregate_publication.py +++ b/tests/integration/test_source_schema_aggregate_publication.py @@ -30,7 +30,7 @@ ) from openhcs.core.memory import numpy as numpy_function from openhcs.core.orchestrator.execution_result import ( - ReturnAllRecords, + MergeAllRuntimeRecords, ) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline.function_contracts import artifact_outputs @@ -115,7 +115,7 @@ def test_real_z_aggregation_persists_scalar_and_unchanged_source_planes(tmp_path orchestrator = PipelineOrchestrator(source, pipeline_config=document.pipeline_config).initialize() outcomes = orchestrator.execute_compiled_bundle( execution_bundle=bundle, max_workers=1, - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, progress_queue=AgentProgressQueue(), progress_context=ProgressExecutionContext( execution_id="source-schema-aggregate-synthetic", diff --git a/tests/unit/test_compiled_execution.py b/tests/unit/test_compiled_execution.py index dbc7c7f375..bf00338628 100644 --- a/tests/unit/test_compiled_execution.py +++ b/tests/unit/test_compiled_execution.py @@ -33,9 +33,9 @@ from openhcs.core.orchestrator import worker_execution from openhcs.core.orchestrator.execution_result import ( ExecutionSucceeded, - ReturnAllRecords, - ReturnDatasetInputRecords, - ReturnNoRecords, + MergeAllRuntimeRecords, + MergeDatasetInputRecords, + OmitRuntimeRecords, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -118,13 +118,13 @@ def test_compiled_execution_bundle_derives_axis_ids_from_contexts() -> None: @pytest.mark.parametrize( ("execution_scopes", "expected_mode"), ( - ((FunctionStepExecutionScope.AXIS,), ReturnNoRecords), + ((FunctionStepExecutionScope.AXIS,), OmitRuntimeRecords), ( ( FunctionStepExecutionScope.AXIS, FunctionStepExecutionScope.PLATE, ), - ReturnDatasetInputRecords, + MergeDatasetInputRecords, ), ), ) @@ -203,18 +203,18 @@ def test_consolidation_does_not_require_parent_payload_records( assert bundle.requires_parent_runtime_observation is False assert ( RuntimeObservationMode.for_compiled_bundle(bundle) - is ReturnNoRecords + is OmitRuntimeRecords ) def test_runtime_observation_mode_can_only_be_strengthened() -> None: assert ( - ReturnNoRecords.including_parent_requirement(True) - is ReturnAllRecords + OmitRuntimeRecords.including_parent_requirement(True) + is MergeAllRuntimeRecords ) assert ( - ReturnAllRecords.including_parent_requirement(False) - is ReturnAllRecords + MergeAllRuntimeRecords.including_parent_requirement(False) + is MergeAllRuntimeRecords ) @@ -262,7 +262,7 @@ def execute_axis(_pipeline, context, _lane, **_kwargs): worker_slot="worker", worker_assignments={"worker": [context.axis_id for context in contexts]}, ), - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, release_axis_resources=False, ) @@ -354,10 +354,10 @@ def execute_current_run( *, context_key, cancellation=None, - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, ): assert context_key == "A01" - assert runtime_observation_mode is ReturnAllRecords + assert runtime_observation_mode is MergeAllRuntimeRecords current_records.append( context.runtime_value_store.replace( measurement_value(2), @@ -395,7 +395,7 @@ def execute_current_run( worker_slot="worker-0", worker_assignments={"worker-0": ["A01"]}, ), - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, ) assert result.runtime_observation.contexts == ( @@ -460,7 +460,7 @@ def fail_reset() -> None: worker_slot="worker-0", worker_assignments={"worker-0": ["A01"]}, ), - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, ) assert cleaned == [(MemoryType.CUPY, 2)] @@ -512,7 +512,7 @@ def execute_axis(_pipeline, context, _lane, **_kwargs): @pytest.mark.parametrize( "observation_mode", - (ReturnNoRecords, ReturnDatasetInputRecords), + (OmitRuntimeRecords, MergeDatasetInputRecords), ) @pytest.mark.parametrize("release_process_resources", (True, False)) def test_worker_lane_releases_unconsumed_image_records_before_next_axis( diff --git a/tests/unit/test_completed_output_publication_lifecycle.py b/tests/unit/test_completed_output_publication_lifecycle.py index 4cb02cc27f..4933475b43 100644 --- a/tests/unit/test_completed_output_publication_lifecycle.py +++ b/tests/unit/test_completed_output_publication_lifecycle.py @@ -19,7 +19,7 @@ from openhcs.core.orchestrator.execution_result import ( ExecutionCancelled, ExecutionFailed, - ReturnNoRecords, + OmitRuntimeRecords, ) from openhcs.core.orchestrator.worker_lanes import WorkerLaneExecutionContext from openhcs.core.runtime_image_values import ImagePayloadMetadata @@ -110,7 +110,7 @@ def process(self, context, index): context = _context(tmp_path, 2) result = worker_execution._execute_axis_with_sequential_combinations( - [Save(), Fail()], [("A01", context)], _lane(), ReturnNoRecords, + [Save(), Fail()], [("A01", context)], _lane(), OmitRuntimeRecords, cancellation=signal, release_axis_resources=False, ) assert type(result) is (ExecutionCancelled if cancel else ExecutionFailed) diff --git a/tests/unit/test_function_artifact_materialization.py b/tests/unit/test_function_artifact_materialization.py index 30724c0842..55c8cd4439 100644 --- a/tests/unit/test_function_artifact_materialization.py +++ b/tests/unit/test_function_artifact_materialization.py @@ -66,8 +66,8 @@ from openhcs.core.post_execute import PostExecuteHook from openhcs.domains.microscopy.analysis_consolidation import AnalysisConsolidationHook from openhcs.core.orchestrator.execution_result import ( - ReturnAllRecords, - ReturnNoRecords, + MergeAllRuntimeRecords, + OmitRuntimeRecords, RuntimeContextObservation, RuntimeExecutionObservation, ) @@ -2007,7 +2007,7 @@ def test_consolidation_preserves_export_bundle_and_text_tables(options, payload) @pytest.mark.parametrize( "observation_mode", - (ReturnNoRecords, ReturnAllRecords), + (OmitRuntimeRecords, MergeAllRuntimeRecords), ) def test_completed_observation_projects_tables_before_worker_payload_release( observation_mode, monkeypatch, @@ -2069,10 +2069,10 @@ def test_completed_observation_projects_tables_before_worker_payload_release( context.runtime_value_store.clear() del pixels assert (pixel_reference() is not None) is ( - observation_mode is ReturnAllRecords + observation_mode is MergeAllRuntimeRecords ) assert len(observation.records) == ( - 2 if observation_mode is ReturnAllRecords else 0 + 2 if observation_mode is MergeAllRuntimeRecords else 0 ) assert len(observation.outputs.runtime_export_paths) == 1 assert observation.outputs.runtime_export_paths[0].suffix == ".csv" diff --git a/tests/unit/test_function_step_execution_scope.py b/tests/unit/test_function_step_execution_scope.py index 27fb88c049..b7d3bb53af 100644 --- a/tests/unit/test_function_step_execution_scope.py +++ b/tests/unit/test_function_step_execution_scope.py @@ -46,9 +46,9 @@ validate_dataset_scoped_contexts, ) from openhcs.core.orchestrator.execution_result import ( - ReturnAllRecords, - ReturnDatasetInputRecords, - ReturnNoRecords, + MergeAllRuntimeRecords, + MergeDatasetInputRecords, + OmitRuntimeRecords, ) from openhcs.core.progress import ProgressEvent, ProgressExecutionContext from openhcs.core.pipeline.function_contracts import ( @@ -688,7 +688,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): count=3, ) observations = context.runtime_value_store.observed_values - retained = ReturnDatasetInputRecords.retain_records( + retained = MergeDatasetInputRecords.retain_records( observations, context, ) @@ -696,13 +696,13 @@ def export(*, artifact_batch: RuntimeArtifactBatch): (["raw"] if stored_image else []) + ["Measurements", "Measurements"] ) assert ( - ReturnAllRecords.retain_records( + MergeAllRuntimeRecords.retain_records( observations, context, ) is observations ) - assert ReturnNoRecords.retain_records(observations, context) == () + assert OmitRuntimeRecords.retain_records(observations, context) == () full_store = type(context.runtime_value_store)() full_store.merge_observed_values(observations) @@ -756,7 +756,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): metadata_writer=True, ) context = _plate_context("A01", (plan,)) - assert ReturnDatasetInputRecords.retain_records((), context) == () + assert MergeDatasetInputRecords.retain_records((), context) == () with pytest.raises(ValueError, match="missing required input"): _dataset_artifact_batch( compiled_contexts={"A01": context}, diff --git a/tests/unit/test_nonpersistent_viewer_state_evidence.py b/tests/unit/test_nonpersistent_viewer_state_evidence.py index d715ad54b5..3435bee563 100644 --- a/tests/unit/test_nonpersistent_viewer_state_evidence.py +++ b/tests/unit/test_nonpersistent_viewer_state_evidence.py @@ -15,7 +15,7 @@ ) from openhcs.core.orchestrator.execution_result import ( ExecutionSucceeded, - ReturnNoRecords, + OmitRuntimeRecords, ) from openhcs.runtime.viewer_protocol import ViewerControlResponse from openhcs.runtime.zmq_server_hooks import ZMQResultsSummaryEnricher @@ -90,7 +90,7 @@ def execute_compiled_bundle(self, **kwargs): execution_id="exec", orchestrator=Orchestrator(), execution_bundle=execution_bundle, - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, progress_context=ProgressExecutionContext( execution_id="exec", subject_id="plate", @@ -101,7 +101,7 @@ def execute_compiled_bundle(self, **kwargs): ).execute() assert returned is compiled_results - assert execution_calls[0]["runtime_observation_mode"] is ReturnNoRecords + assert execution_calls[0]["runtime_observation_mode"] is OmitRuntimeRecords assert progress_queue.values == [None] assert record.get_extra(BundleExecutionExtras.EXECUTION_RECORD_KEY) is ( compiled_results.extras diff --git a/tests/unit/test_orchestrator_lane_planning.py b/tests/unit/test_orchestrator_lane_planning.py index c80ae8cf07..df50215047 100644 --- a/tests/unit/test_orchestrator_lane_planning.py +++ b/tests/unit/test_orchestrator_lane_planning.py @@ -48,8 +48,8 @@ ExecutionCancelled, ExecutionFailed, ExecutionSucceeded, - ReturnAllRecords, - ReturnNoRecords, + MergeAllRuntimeRecords, + OmitRuntimeRecords, RuntimeExecutionObservation, ) from openhcs.core.orchestrator.worker_execution import ( @@ -123,7 +123,7 @@ def test_fork_resources_reuse_lane_processes_for_ordered_plate_partitions(monkey worker_assignments={"worker_0": ["A01"], "worker_1": ["A02"]}, lane_axis_contexts={"worker_0": [("A01", [])], "worker_1": [("A02", [])]}, ), - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, ) processes = [] try: @@ -180,7 +180,7 @@ def test_fork_resources_join_all_lanes_after_plate_failure(monkeypatch, failure) worker_assignments={"worker_0": ["A01"], "worker_1": ["A02"]}, lane_axis_contexts={"worker_0": [("A01", [])], "worker_1": [("A02", [])]}, ), - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, ) processes = [] try: @@ -253,7 +253,7 @@ def _execute_with_visualizer(monkeypatch, visualizer, *, progress_queue=None): visualizer=None, log_file_base=None, progress_queue=progress_queue or SimpleNamespace(put=lambda _event: None), - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, debug_execution_policy=NoOpDebugExecutionPolicy(), ) @@ -390,7 +390,7 @@ def test_compiled_plate_execution_request_uses_bundle_as_runtime_authority(): visualizer=None, log_file_base=None, progress_queue="queue", - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, debug_execution_policy=NoOpDebugExecutionPolicy(), ) orchestrator = SimpleNamespace( @@ -427,7 +427,7 @@ def test_plate_scope_rejects_omitted_runtime_observations(monkeypatch): visualizer=None, log_file_base=None, progress_queue="queue", - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, debug_execution_policy=NoOpDebugExecutionPolicy(), ) monkeypatch.setattr( @@ -770,7 +770,7 @@ def held_lane(**_kwargs): writer, (reader,), worker_execution_module.ExecuteInheritedLane( - [], lane, ReturnNoRecords + [], lane, OmitRuntimeRecords ), ), ) @@ -840,7 +840,7 @@ def fake_execute_worker_lane( worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", [("A01", object())])]}, ), - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, ) results = PooledWorkerLaneRunner(FakeExecutor(), cancellation=None).run( @@ -891,7 +891,7 @@ def execute_axis( pipeline_definition=[], lane_axis_contexts=lane_axis_contexts, lane_context=lane_context, - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, cancellation=cancellation, ) @@ -965,7 +965,7 @@ def fake_execute_worker_lane( worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", [("A01", object())])]}, ), - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, ) PooledWorkerLaneRunner(FakeExecutor(), cancellation=None).run( @@ -1033,7 +1033,7 @@ def fake_execute_worker_lane( worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", ["A01"])]}, ), - runtime_observation_mode=ReturnAllRecords, + runtime_observation_mode=MergeAllRuntimeRecords, ) results = ForkInheritedWorkerLaneRunner(ForbiddenMultiprocessingContext()).run( @@ -1074,7 +1074,7 @@ def submit(self, fn, *args): worker_assignments={"worker_0": ["A01"]}, lane_axis_contexts={"worker_0": [("A01", [("A01", object())])]}, ), - runtime_observation_mode=ReturnNoRecords, + runtime_observation_mode=OmitRuntimeRecords, ) with pytest.raises(RuntimeError, match="lane exploded"): diff --git a/tests/unit/test_zmq_execution_server.py b/tests/unit/test_zmq_execution_server.py index 863bd343de..051da4e032 100644 --- a/tests/unit/test_zmq_execution_server.py +++ b/tests/unit/test_zmq_execution_server.py @@ -29,9 +29,9 @@ from openhcs.core.orchestrator.execution_result import ( ExecutionFailed, ExecutionSucceeded, - ReturnAllRecords, - ReturnDatasetInputRecords, - ReturnNoRecords, + MergeAllRuntimeRecords, + MergeDatasetInputRecords, + OmitRuntimeRecords, RuntimeContextObservation, RuntimeExecutionObservation, ) @@ -134,7 +134,7 @@ def test_axis_outcomes_qualify_server_terminal_after_export(monkeypatch, failed) compile_only=False, execution_id=record.execution_id, auxiliary_params=SimpleNamespace( - runtime_observation_mode_for=lambda bundle: ReturnNoRecords + runtime_observation_mode_for=lambda bundle: OmitRuntimeRecords ), ) monkeypatch.setattr( @@ -225,26 +225,26 @@ def test_terminal_notification_contains_openhcs_summary_before_cleanup( @pytest.mark.parametrize( ("compiled_mode", "export_path", "expected_mode"), ( - (ReturnNoRecords, None, ReturnNoRecords), + (OmitRuntimeRecords, None, OmitRuntimeRecords), ( - ReturnNoRecords, + OmitRuntimeRecords, "/tmp/runtime-observation.pkl", - ReturnAllRecords, + MergeAllRuntimeRecords, ), ( - ReturnDatasetInputRecords, + MergeDatasetInputRecords, None, - ReturnDatasetInputRecords, + MergeDatasetInputRecords, ), ( - ReturnDatasetInputRecords, + MergeDatasetInputRecords, "/tmp/runtime-observation.pkl", - ReturnAllRecords, + MergeAllRuntimeRecords, ), ( - ReturnAllRecords, + MergeAllRuntimeRecords, None, - ReturnDatasetInputRecords, + MergeDatasetInputRecords, ), ), ) @@ -276,7 +276,7 @@ def test_outcome_export_does_not_strengthen_worker_runtime_value_retention() -> assert ( params.runtime_observation_mode_for(execution_bundle) - is ReturnNoRecords + is OmitRuntimeRecords ) assert ZMQAuxiliaryExecutionParams.from_transport(params.to_transport()) == params From f958e92c5308d47757f1e432fdcccb2b3cef16af Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 15:05:10 -0400 Subject: [PATCH 04/13] WIP: K7 component-group scope variants, plan group_scope field, compose split Co-Authored-By: Claude Opus 5.5 --- .../artifact_plan_inspection_service.py | 4 +- openhcs/core/artifacts/input_projection.py | 92 ++-- openhcs/core/artifacts/kinds.py | 61 ++- openhcs/core/artifacts/plans.py | 116 ++-- openhcs/core/compiled_step_plan.py | 7 +- openhcs/core/component_group_scope.py | 503 ++++++++++++------ openhcs/core/debug.py | 2 +- openhcs/core/pipeline/compiler.py | 17 +- openhcs/core/pipeline/path_planner.py | 395 +++++--------- openhcs/core/runtime_artifact_values.py | 83 ++- openhcs/core/runtime_stores.py | 94 ++-- .../function_artifact_materialization.py | 19 +- openhcs/core/steps/function_execution.py | 8 +- .../runtime/output_record_request.py | 4 +- .../pyqt_gui/widgets/artifact_plan_view.py | 2 +- .../test_chained_callable_lineage_journey.py | 2 +- tests/unit/agent/test_agent_services.py | 8 +- tests/unit/progress/test_live_measurements.py | 6 +- .../unit/test_aligned_main_flow_projection.py | 6 +- .../test_artifact_input_edge_cardinality.py | 22 +- .../test_artifact_input_projection_plan.py | 54 +- tests/unit/test_artifact_plan_view.py | 5 +- tests/unit/test_cellprofiler_area_occupied.py | 5 +- ...test_cellprofiler_artifact_declarations.py | 5 +- ...ofiler_artifact_identity_reconstruction.py | 7 +- ...ellprofiler_combineobjects_source_scope.py | 11 +- ...lprofiler_generic_special_input_binding.py | 11 +- ...test_cellprofiler_library_untangleworms.py | 5 +- ...ellprofiler_measurement_slice_consumers.py | 5 +- ...profiler_measurement_source_group_scope.py | 8 +- .../test_cellprofiler_module_execution.py | 80 ++- .../unit/test_cellprofiler_runtime_adapter.py | 396 +++++++------- ..._cellprofiler_runtime_cache_authorities.py | 13 +- ...profiler_runtime_input_binding_contract.py | 29 +- .../test_cellprofiler_spreadsheet_export.py | 6 +- ...st_child_measurement_producer_selection.py | 31 +- .../test_classification_interop_boundaries.py | 5 +- tests/unit/test_compiled_execution.py | 9 +- tests/unit/test_debug_runtime.py | 21 +- .../unit/test_derived_role_publication_435.py | 4 +- .../unit/test_funcstep_contract_validator.py | 19 +- .../test_function_artifact_materialization.py | 54 +- tests/unit/test_function_artifact_outputs.py | 27 +- .../test_function_core_executor_ownership.py | 7 +- ...st_function_execution_lifecycle_anchors.py | 16 +- ...st_function_output_manifest_input_scope.py | 4 +- tests/unit/test_function_patterns.py | 42 +- ...test_function_runtime_source_projection.py | 102 ++-- .../unit/test_function_step_execution_plan.py | 47 +- .../test_function_step_execution_scope.py | 14 +- .../test_implicit_main_flow_compilation.py | 7 +- ...nvocation_input_source_context_identity.py | 7 +- .../test_loaded_pattern_group_ownership.py | 16 +- .../test_opaque_main_flow_image_domain.py | 7 +- .../unit/test_output_publication_ownership.py | 7 +- .../unit/test_path_planner_materialization.py | 474 ++++++++--------- tests/unit/test_payload_axis_compile_gate.py | 4 +- tests/unit/test_runtime_artifact_queries.py | 6 +- tests/unit/test_runtime_output_matching.py | 12 +- tests/unit/test_runtime_value_store.py | 240 ++++----- tests/unit/test_runtime_values.py | 23 +- ..._source_bound_matched_image_set_anchors.py | 30 +- 62 files changed, 1615 insertions(+), 1711 deletions(-) diff --git a/openhcs/agent/services/artifact_plan_inspection_service.py b/openhcs/agent/services/artifact_plan_inspection_service.py index af7b08c252..dc50f2ba0c 100644 --- a/openhcs/agent/services/artifact_plan_inspection_service.py +++ b/openhcs/agent/services/artifact_plan_inspection_service.py @@ -461,7 +461,7 @@ def _artifact_input_summary(plan) -> ArtifactInputPlanSummary: name=str(plan.name), kind=str(plan.artifact_type.value), path=str(plan.path), - group_keys=tuple(plan.group_keys), + group_keys=plan.group_scope.keys, paths_by_group=_artifact_paths_by_group(plan), source_step_id=plan.source_step_id, source_step_scope_id=plan.source_step_scope_id, @@ -478,7 +478,7 @@ def _artifact_summary( name=str(plan.name), kind=str(plan.artifact_type.value), path=str(plan.path), - group_keys=tuple(plan.group_keys), + group_keys=plan.group_scope.keys, paths_by_group=_artifact_paths_by_group(plan), materialization=_artifact_materialization_summary( context, diff --git a/openhcs/core/artifacts/input_projection.py b/openhcs/core/artifacts/input_projection.py index 7ce2b7da5d..df3a40404b 100644 --- a/openhcs/core/artifacts/input_projection.py +++ b/openhcs/core/artifacts/input_projection.py @@ -31,7 +31,7 @@ def declared_producer_selection_scope( ) -> ComponentGroupScope | None: """Resolve one producer selection declared by the input's relations.""" - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope scopes = input_spec.relation_answers( lambda relation: relation.input_producer_selection_scope(producer_scope) ) @@ -60,23 +60,23 @@ def producer_selection_for_invocation( declared = cls.declared_producer_selection_scope(input_spec, storage_plan) if declared is not None: return declared - producer_scope = storage_plan.producer_group_scope() - if producer_scope.is_ungrouped: - return producer_scope - if ( - producer_scope.has_single_static_key - or execution_scope is FunctionStepExecutionScope.PLATE - or producer_scope.component in consumer_variable_components - ): - return producer_scope - for scope in component_scopes: - if scope.component is producer_scope.component: - return scope - raise ValueError( - f"Invocation {invocation_key!r} input {storage_plan.ref()!r} has " - f"producer scope {producer_scope!r} but no exact relation-owned " - "selection coordinate." - ) + producer_scope = storage_plan.group_scope + for grouped in producer_scope.as_grouped(): + if ( + grouped.single_key() is not None + or execution_scope is FunctionStepExecutionScope.PLATE + or grouped.component in consumer_variable_components + ): + return grouped + for scope in component_scopes: + if scope.component is grouped.component: + return scope + raise ValueError( + f"Invocation {invocation_key!r} input {storage_plan.ref()!r} has " + f"producer scope {grouped!r} but no exact relation-owned " + "selection coordinate." + ) + return producer_scope def selects_declared_complete_producer( self, input_spec: ArtifactSpec, storage_plan: ArtifactInputPlan @@ -91,25 +91,15 @@ def selects_declared_complete_producer( f"Artifact input {input_spec.ref()!r} compiled selection " f"{self.producer_selection_scope!r} contradicts declaration {declared!r}." ) - return declared == storage_plan.producer_group_scope() + return declared == storage_plan.group_scope def __post_init__(self) -> None: - object.__setattr__( - self, - "invocation_scope", - ComponentGroupScope.from_raw( - self.invocation_scope.keys, - component=self.invocation_scope.component, - ), - ) - object.__setattr__( - self, - "producer_selection_scope", - ComponentGroupScope.from_raw( - self.producer_selection_scope.keys, - component=self.producer_selection_scope.component, - ), - ) + for scope in (self.invocation_scope, self.producer_selection_scope): + if not isinstance(scope, ComponentGroupScope): + raise TypeError( + "Artifact input projection scopes must be ComponentGroupScope, " + f"got {type(scope).__name__}." + ) object.__setattr__( self, "component_scopes", @@ -150,22 +140,9 @@ def validate_storage_plan(self, storage_plan: ArtifactInputPlan) -> None: """Require this projection to select within its producer storage plan.""" artifact_ref = storage_plan.ref() - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope selection_scope = self.producer_selection_scope - if producer_scope.is_ungrouped: - if selection_scope.is_ungrouped: - return - raise ValueError( - f"Ungrouped producer {artifact_ref!r} cannot use grouped " - f"selection {selection_scope!r}." - ) - if selection_scope.component is not producer_scope.component: - raise ValueError( - f"Artifact input {artifact_ref!r} producer component " - f"{producer_scope.component.name!r} does not match selection " - f"{selection_scope.component!r}." - ) - if selection_scope.is_dynamic or producer_scope.contains_scope(selection_scope): + if selection_scope.selectable_from(producer_scope): return raise ValueError( f"Artifact input {artifact_ref!r} selection " @@ -178,21 +155,16 @@ def validate_axis_projection(self, storage_plan: ArtifactInputPlan) -> None: self.validate_storage_plan(storage_plan) artifact_ref = storage_plan.ref() - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope selection_scope = self.producer_selection_scope retained_components = ComponentSet.of(self.consumer_variable_components) - # validate_storage_plan proved an ungrouped producer has an ungrouped - # selection, so only grouped producers reach either branch below. if producer_scope.component in retained_components: if selection_scope != producer_scope: raise ValueError( f"Retained producer axis {producer_scope.component.name!r} for " f"{artifact_ref!r} must select the complete producer scope." ) - elif selection_scope.is_dynamic and ( - self.invocation_scope.component is not selection_scope.component - or self.invocation_scope.is_ungrouped - ): + elif not selection_scope.owned_by(self.invocation_scope): raise ValueError( f"Dynamic producer selection {selection_scope!r} for " f"{artifact_ref!r} is not owned by invocation scope " @@ -206,12 +178,12 @@ def validate_axis_projection(self, storage_plan: ArtifactInputPlan) -> None: f"Artifact input {artifact_ref!r} projects producer stack " f"component {component.name!r} without an exact coordinate." ) - if scope.is_static and len(scope.keys) != 1: + if scope.names_several_keys: raise ValueError( f"Artifact input {artifact_ref!r} projection for component " f"{component.name!r} is not a single exact coordinate: {scope!r}." ) - if scope.is_dynamic and self.invocation_scope.component is not component: + if not scope.owned_by(self.invocation_scope): raise ValueError( f"Dynamic stack projection {scope!r} for {artifact_ref!r} " f"is not owned by invocation scope {self.invocation_scope!r}." @@ -225,7 +197,7 @@ def validate_complete_producer_projection( self.validate_storage_plan(storage_plan) artifact_ref = storage_plan.ref() - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope if self.producer_selection_scope != producer_scope: raise ValueError( f"Plate artifact input {artifact_ref!r} must select complete " diff --git a/openhcs/core/artifacts/kinds.py b/openhcs/core/artifacts/kinds.py index b4ff37180c..93cc3609fb 100644 --- a/openhcs/core/artifacts/kinds.py +++ b/openhcs/core/artifacts/kinds.py @@ -24,7 +24,6 @@ TablePayloadShape, ) from openhcs.core.axes import AxisFamily -from openhcs.core.component_group_scope import ComponentGroupScope from openhcs.core.memory import detect_memory_type from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, @@ -383,11 +382,9 @@ def source_image_payload_from_runtime_value(cls, value: object) -> object | None def compose_runtime_values( cls, values: Sequence[RuntimeValue], - producer_group_scope: ComponentGroupScope | None = None, ) -> object: """Compose producer records according to this artifact's semantics.""" - del producer_group_scope if len(values) != 1: raise ValueError( f"{cls.__name__} does not define grouped runtime composition for " @@ -395,6 +392,12 @@ def compose_runtime_values( ) return values[0].data + @classmethod + def compose_runtime_groups(cls, values: Sequence[RuntimeValue]) -> object: + """Rebuild a producer's group axis from one record per group.""" + + return cls.compose_runtime_values(values) + @classmethod def normalize_slice_aligned_value( cls, @@ -836,13 +839,26 @@ def named_payload(payload: object) -> object: def compose_runtime_values( cls, values: Sequence[RuntimeValue], - producer_group_scope: ComponentGroupScope | None = None, ) -> object: - """Stack grouped image records while preserving payload context.""" + """Stack image records while preserving payload context.""" - if producer_group_scope is None and len(values) == 1: + if len(values) == 1: return values[0].data + return cls._stack_runtime_values(values, keeps_memory_type=False) + + @classmethod + def compose_runtime_groups(cls, values: Sequence[RuntimeValue]) -> object: + """Rebuild the producer's group axis on the producer's memory type.""" + return cls._stack_runtime_values(values, keeps_memory_type=True) + + @classmethod + def _stack_runtime_values( + cls, + values: Sequence[RuntimeValue], + *, + keeps_memory_type: bool, + ) -> object: from openhcs.core.aligned_image_payload import stack_image_payloads from openhcs.core.runtime_image_values import ImagePayloadMetadataCompositionMode @@ -856,7 +872,7 @@ def compose_runtime_values( metadata_mode = ImagePayloadMetadataCompositionMode.for_plane_axis( RuntimePlaneAxis.RUNTIME_SLICE ) - if producer_group_scope is None: + if not keeps_memory_type: return stack_image_payloads(payloads, metadata_mode=metadata_mode) return stack_image_payloads( payloads, @@ -881,7 +897,7 @@ def normalize_group_scoped_payload( if projection.axis_size != 1: raise ValueError( f"Group-scoped scalar artifact {output_plan.name!r} for " - f"{output_plan.group_component.name!r} cannot retain a declared " + f"{output_plan.group_scope.component.name!r} cannot retain a declared " f"runtime-slice axis of size {projection.axis_size}." ) return RuntimeSliceProjection.value_for_slice( @@ -1094,12 +1110,17 @@ def _aggregated(cls, name: str, labels: tuple[object, ...]) -> object: def compose_runtime_values( cls, values: Sequence[RuntimeValue], - producer_group_scope: ComponentGroupScope | None = None, ) -> object: - """Aggregate grouped object-label records into one runtime stack.""" + """Aggregate object-label records into one runtime stack.""" - if producer_group_scope is None and len(values) == 1: + if len(values) == 1: return values[0].data + return cls.compose_runtime_groups(values) + + @classmethod + def compose_runtime_groups(cls, values: Sequence[RuntimeValue]) -> object: + """Aggregate grouped object-label records into one runtime stack.""" + labels = tuple(value.data for value in values) if len({value.representation for value in labels}) != 1: raise ValueError( @@ -1301,13 +1322,11 @@ def runtime_semantic_id(cls, data: object) -> str | None: def compose_runtime_values( cls, values: Sequence[RuntimeValue], - producer_group_scope: ComponentGroupScope | None = None, ) -> object: """Concatenate exact producer-group measurement tables.""" from openhcs.core.runtime_measurements import MeasurementTable - del producer_group_scope return MeasurementTable.join( values[0].name, tuple(value.data for value in values) ) @@ -1385,14 +1404,18 @@ def runtime_payload_type(cls) -> type: def compose_runtime_values( cls, values: Sequence[RuntimeValue], - producer_group_scope: ComponentGroupScope | None = None, ) -> object: """Reconstruct relationship records and retain grouped slice alignment.""" - relationships = tuple(value.data for value in values) - if producer_group_scope is None and len(relationships) == 1: - return relationships[0] - return RuntimeSliceAlignedValues(relationships) + if len(values) == 1: + return values[0].data + return cls.compose_runtime_groups(values) + + @classmethod + def compose_runtime_groups(cls, values: Sequence[RuntimeValue]) -> object: + """Keep each producer group as one slice of an aligned value.""" + + return RuntimeSliceAlignedValues(tuple(value.data for value in values)) class RelationshipsArtifactType(ObjectLineageArtifactType): @@ -1510,13 +1533,11 @@ def grid(item: object) -> SpatialGrid: def compose_runtime_values( cls, values: Sequence[RuntimeValue], - producer_group_scope: ComponentGroupScope | None = None, ) -> object: """Compose only spatial grids with identical declared slice domains.""" from openhcs.core.runtime_slice_projection import RuntimeSliceProjection - del producer_group_scope grids = tuple(value.data for value in values) def layout(grid_value: object) -> tuple[object, ...]: diff --git a/openhcs/core/artifacts/plans.py b/openhcs/core/artifacts/plans.py index 1858c56622..5c2772b0de 100644 --- a/openhcs/core/artifacts/plans.py +++ b/openhcs/core/artifacts/plans.py @@ -25,7 +25,7 @@ from openhcs.core.artifacts.sidecars import ArtifactSidecarRole from openhcs.core.artifacts.streaming import AutomaticStreaming, ViewerStreaming from openhcs.core.axes import Axis -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ComponentGroupScope, UngroupedScope from openhcs.core.component_set import ComponentSet if TYPE_CHECKING: @@ -285,8 +285,7 @@ class ArtifactPlan(DeclaredRelations, ABC, metaclass=AutoRegisterMeta): name: str path: str artifact_type: type[ArtifactType] = SpecialArtifactType - group_keys: tuple[str | None, ...] = (None,) - group_component: type[Axis] | None = None + group_scope: ComponentGroupScope = field(default_factory=UngroupedScope) variable_components: tuple[type[Axis], ...] = () component_domains: tuple[ComponentGroupScope, ...] = () paths_by_group: Mapping[str | None, str] | None = None @@ -314,21 +313,18 @@ def __post_init__(self) -> None: owner=f"Artifact plan {self.name!r}", ), ) - if self.group_component in self.variable_components: - raise ValueError( - f"Artifact plan {self.name!r} cannot group by " - f"{self.group_component.name!r} while also retaining it as a " - "variable component." + if not isinstance(self.group_scope, ComponentGroupScope): + raise TypeError( + f"Artifact plan {self.name!r} group_scope must be a " + f"ComponentGroupScope, got {type(self.group_scope).__name__}." ) - self.group_scope() - - def group_scope(self) -> ComponentGroupScope: - """Return this plan's typed component-group identity.""" - - return ComponentGroupScope.from_raw( - self.group_keys or (None,), - component=self.group_component, - ) + for grouped in self.group_scope.as_grouped(): + if grouped.component in self.variable_components: + raise ValueError( + f"Artifact plan {self.name!r} cannot group by " + f"{grouped.component.name!r} while also retaining it as a " + "variable component." + ) @property def participates_in_main_flow(self) -> bool: @@ -336,6 +332,15 @@ def participates_in_main_flow(self) -> bool: return self.artifact_type.publishes_to_main_flow(self.sidecar_role) + def relation_scope( + self, + component: type[Axis] | None, + ) -> ComponentGroupScope: + """Return the domain this plan carries for one relation component.""" + + domain = None if component is None else self.component_domain(component) + return self.group_scope if domain is None else domain + def component_domain( self, component: type[Axis], @@ -357,8 +362,7 @@ def component_coordinate_scopes(self) -> tuple[ComponentGroupScope, ...]: scopes_by_component = { domain.component: domain for domain in self.component_domains } - group_scope = self.group_scope() - if group_scope.is_grouped: + for group_scope in self.group_scope.as_grouped(): inherited_domain = scopes_by_component.get(group_scope.component) if inherited_domain is not None and not inherited_domain.contains_scope( group_scope @@ -389,26 +393,14 @@ def is_component_coordinate_disjoint_from(self, other: ArtifactPlan) -> bool: if own_scope.component in other_scopes ) - @property - def single_group_key(self) -> str | None: - group_keys = self.group_keys or (None,) - if len(group_keys) == 1: - return group_keys[0] - return None - - @property - def has_dynamic_group_scope(self) -> bool: - """Return whether concrete runtime groups are discovered during execution.""" - return self.group_scope().is_dynamic - def require_single_group_key(self) -> str | None: """Return the only artifact group key, failing for ambiguous groups.""" - group_keys = self.group_keys or (None,) - if len(group_keys) == 1: - return group_keys[0] - raise RuntimeError( - f"Artifact plan '{self.name}' requires one group key, got {group_keys!r}." - ) + if self.group_scope.names_several_keys: + raise RuntimeError( + f"Artifact plan '{self.name}' requires one group key, got " + f"{self.group_scope.keys!r}." + ) + return self.group_scope.single_key() def ref(self) -> ArtifactSpecRef: """Return this compiled plan's scope-free declaration identity.""" @@ -450,10 +442,11 @@ def _path_for_group(self, group_key: str | None) -> str | None: if group_key in self.paths_by_group: return self.paths_by_group[group_key] if None in self.paths_by_group: - if group_key is None or self.group_component is None: + component = self.group_scope.component + if group_key is None or component is None: return self.paths_by_group[None] return grouped_artifact_path( - self.paths_by_group[None], self.group_component, group_key + self.paths_by_group[None], component, group_key ) if self._missing_group_uses_default_path: return self.path @@ -468,7 +461,9 @@ def _plan_for_group(self, group_key: str | None) -> Self | None: replace( self, path=group_path, - group_keys=(group_key,), + group_scope=ComponentGroupScope.of( + (group_key,), self.group_scope.component + ), paths_by_group={group_key: group_path}, ), ) @@ -553,7 +548,7 @@ def normalize_payload(self, value: object, *, axis_id: str) -> object: aligned, self, axis_id=axis_id ), ) - if self.group_component is None or self.variable_components: + if not self.group_scope.as_grouped() or self.variable_components: return normalized return kind.normalize_group_scoped_payload(self, normalized) @@ -602,7 +597,7 @@ def group_scope_sources(self) -> tuple[ArtifactSpecRef, ...]: declared = super().group_scope_sources() source_map = self.group_scope_sources_by_group - group_keys = self.group_scope().keys + group_keys = self.group_scope.keys if not source_map or len(group_keys) != 1: return declared selected = source_map.get(group_keys[0]) @@ -623,10 +618,12 @@ def for_invocation_group( ) -> ArtifactOutputPlan: """Resolve storage scope from the invocation or this exact output plan.""" - output_scope = self.group_scope() - if group_key is None and output_scope.is_static: - group_key = output_scope.require_single_static_key() - return self.for_group(output_scope.resolve_runtime_key(group_key)) + scope = self.group_scope + return self.for_group( + scope.select_runtime_key(None) + if group_key is None + else scope.resolve_runtime_key(group_key) + ) def for_execution_scope( self, @@ -635,15 +632,15 @@ def for_execution_scope( ) -> ArtifactOutputPlan | None: """Project this output into one compiled component execution.""" - output_scope = self.group_scope() - if output_scope.is_ungrouped: - return self - if output_scope.component is execution_scope.component: - if output_scope.contains_runtime_key(component_key): - return self.for_group(output_scope.resolve_runtime_key(component_key)) - return None - if output_scope.has_single_static_key: - return self.for_invocation_group(None) + output_scope = self.group_scope + for grouped in output_scope.as_grouped(): + if execution_scope.groups_by(grouped.component): + if grouped.contains_runtime_key(component_key): + return self.for_group(grouped.resolve_runtime_key(component_key)) + return None + single_key = output_scope.single_key() + if single_key is not None: + return self.for_group(single_key) return self @@ -660,18 +657,13 @@ def for_group(self, group_key: str | None) -> ArtifactInputPlan | None: """Return a group-specific input plan, or None if not available.""" return self._plan_for_group(group_key) - def producer_group_scope(self) -> ComponentGroupScope: - """Return the compiler-resolved scope of this named producer.""" - - return self.group_scope() - def composes_producer_groups( self, consumer_variable_components: ComponentSet, ) -> bool: """Return whether this consumer reconstructs the producer group axis.""" - return self.producer_group_scope().component in consumer_variable_components + return self.group_scope.component in consumer_variable_components def retains_producer_stack( self, @@ -692,7 +684,7 @@ def runtime_variable_components( """Return axes present after this input is reconstructed for a consumer.""" if self.composes_producer_groups(consumer_variable_components): - return ComponentSet((self.producer_group_scope().component,)) + return ComponentSet((self.group_scope.component,)) if self.retains_producer_stack(consumer_variable_components): return consumer_variable_components return ComponentSet() diff --git a/openhcs/core/compiled_step_plan.py b/openhcs/core/compiled_step_plan.py index 6e96fccbcf..5232b622b8 100644 --- a/openhcs/core/compiled_step_plan.py +++ b/openhcs/core/compiled_step_plan.py @@ -17,7 +17,10 @@ ArtifactSpecRef, ) from openhcs.core.callable_contract import FunctionStepExecutionScope -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.function_patterns import ( CompiledFunctionGroup, CompiledFunctionPattern, @@ -295,7 +298,7 @@ class CompiledStepPlan: ) future_artifact_inputs: frozenset[ArtifactSpecRef] | None = None execution_group_scope: ComponentGroupScope = field( - default_factory=ComponentGroupScope.ungrouped + default_factory=UngroupedScope ) compiled_function_pattern: CompiledFunctionPattern | None = None input_conversion: InputConversionPlan | None = None diff --git a/openhcs/core/component_group_scope.py b/openhcs/core/component_group_scope.py index b4c92e5829..17e909e6de 100644 --- a/openhcs/core/component_group_scope.py +++ b/openhcs/core/component_group_scope.py @@ -3,8 +3,9 @@ from __future__ import annotations from collections.abc import Hashable, Iterable, Mapping, Sequence +from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from openhcs.core.component_set import ComponentSet from openhcs.core.source_metadata import ( @@ -26,58 +27,49 @@ RuntimeFixedComponentValues = tuple[tuple[type[Axis], str], ...] -@dataclass(frozen=True, slots=True) -class ComponentGroupScope: - """One validated ungrouped, static, or runtime-discovered component scope.""" +class ComponentGroupScope(ABC): + """Which component groups one execution or artifact covers. - keys: tuple[ComponentGroupKey, ...] - component: type[Axis] | None = None + Each variant is a subclass: ``UngroupedScope`` (one default group), + ``DynamicGroupScope`` (groups of one axis discovered at run time), + ``StaticGroupScope`` (several groups named at compile time) and + ``SingleKeyScope`` (exactly one named group). + """ - def __post_init__(self) -> None: - if not self.keys: - raise ValueError("ComponentGroupScope.keys cannot be empty.") - if any(key is not None and not isinstance(key, str) for key in self.keys): - raise TypeError( - "ComponentGroupScope.keys must contain canonical strings or None." - ) - if None in self.keys and self.keys != (None,): - raise ValueError( - "ComponentGroupScope may use None only as the sole dynamic or " - "ungrouped key." - ) - if self.component is not None and not is_axis(self.component): - raise TypeError("ComponentGroupScope.component must be a declared axis.") - if self.component is None and self.keys != (None,): - raise ValueError( - "Concrete component-group keys require a component identity." - ) - - @classmethod - def ungrouped(cls) -> "ComponentGroupScope": - return cls((None,)) - - @classmethod - def dynamic( - cls, - component: type[Axis], - ) -> "ComponentGroupScope": - """Declare a typed scope whose concrete keys are discovered at runtime.""" + __slots__ = () - return cls((None,), component=component) + component: type[Axis] | None + keys: tuple[ComponentGroupKey, ...] + has_compiled_keys: ClassVar[bool] = True + names_several_keys: ClassVar[bool] = False - @classmethod - def from_raw( - cls, + @staticmethod + def of( group_keys: Iterable[Hashable | None], - *, component: type[Axis] | None = None, ) -> "ComponentGroupScope": - return cls( - tuple(cls.normalize_key(group_key) for group_key in group_keys), - component=( - None if component is None else AxisFamily.active().require(component) - ), + """Read a scope from its raw spelling: group keys plus a component.""" + + keys = tuple( + dict.fromkeys(ComponentGroupScope.normalize_key(key) for key in group_keys) ) + if component is None: + if keys not in ((), (None,)): + raise ValueError( + "Concrete component-group keys require a component identity." + ) + return UngroupedScope() + axis = AxisFamily.active().require(component) + if keys in ((), (None,)): + return DynamicGroupScope(axis) + if None in keys: + raise ValueError( + "ComponentGroupScope may use None only as the sole dynamic or " + "ungrouped key." + ) + if len(keys) == 1: + return SingleKeyScope(axis, keys[0]) + return StaticGroupScope(axis, keys) @staticmethod def normalize_key(key: Hashable | None) -> ComponentGroupKey: @@ -85,149 +77,149 @@ def normalize_key(key: Hashable | None) -> ComponentGroupKey: return None return str(key) - @property - def is_ungrouped(self) -> bool: - return self.component is None - - @property - def is_grouped(self) -> bool: - return self.component is not None - - @property - def is_dynamic(self) -> bool: - return self.component is not None and self.keys == (None,) - - @property - def is_static(self) -> bool: - """Whether this grouped scope names its coordinates at compile time.""" - return self.is_grouped and not self.is_dynamic - - def excludes(self, other: "ComponentGroupScope") -> bool: - """Whether two static scopes of one component share no coordinate.""" - if self.is_dynamic or other.is_dynamic: - return False - return set(self.keys).isdisjoint(other.keys) - @classmethod def one_per_component( cls, scopes: Iterable["ComponentGroupScope"], *, owner: str, - ) -> tuple["ComponentGroupScope", ...]: - """Normalize grouped scopes, allowing one consistent scope per component.""" - by_component: dict[type[Axis], ComponentGroupScope] = {} + ) -> tuple["GroupedScope", ...]: + """Allow one consistent grouped scope per component.""" + by_component: dict[type[Axis], GroupedScope] = {} for scope in scopes: - if scope.is_ungrouped: - raise ValueError(f"{owner} component scopes cannot be ungrouped.") - normalized = cls.from_raw(scope.keys, component=scope.component) - existing = by_component.get(scope.component) - if existing is not None and existing != normalized: + grouped = scope.require_grouped(owner) + existing = by_component.get(grouped.component) + if existing is not None and existing != grouped: raise ValueError( f"{owner} declares conflicting scopes for component " - f"{scope.component.name!r}: {existing!r} and {normalized!r}." + f"{grouped.component.name!r}: {existing!r} and {grouped!r}." ) - by_component[scope.component] = normalized + by_component[grouped.component] = grouped return tuple(by_component.values()) - @property - def has_single_static_key(self) -> bool: - """Return whether this scope owns one exact compiler-known coordinate.""" + @staticmethod + def union_compatible( + scopes: Sequence["ComponentGroupScope"], + ) -> "ComponentGroupScope | None": + """Union scopes of one component; None when their components differ.""" - return not self.is_ungrouped and not self.is_dynamic and len(self.keys) == 1 + if not scopes: + return None + component = scopes[0].component + if any(scope.component is not component for scope in scopes[1:]): + return None + if not all(scope.has_compiled_keys for scope in scopes): + return DynamicGroupScope(component) + return ComponentGroupScope.of( + (key for scope in scopes for key in scope.keys), + component, + ) + + def missing_from( + self, + producer_scope: "ComponentGroupScope", + ) -> list[ComponentGroupKey]: + """Return this scope's keys the producer does not plan.""" + return [key for key in self.keys if key not in producer_scope.keys] + + def single_key(self) -> str | None: + """Return the one compiler-known coordinate of a single-key scope.""" + return None + + def require_single_key(self) -> str: + raise ValueError( + f"A single static component key is required, got {self!r}." + ) + + @abstractmethod + def as_grouped(self) -> tuple["GroupedScope", ...]: + """Return this scope when it groups by a component, else nothing.""" + + def require_grouped(self, owner: str) -> "GroupedScope": + raise ValueError(f"{owner} component scopes cannot be ungrouped.") + + def groups_by(self, component: type[Axis] | None) -> bool: + """Whether this scope groups by ``component``.""" + return False + @abstractmethod def runtime_keys( self, discovered_keys: Iterable[Hashable | None], ) -> tuple[ComponentGroupKey, ...]: """Resolve concrete execution keys from this compiler-owned scope.""" - if self.is_ungrouped: - return (None,) - if not self.is_dynamic: - return self.keys - keys = tuple( - dict.fromkeys( - self.normalize_key(key) for key in discovered_keys if key is not None - ) - ) - if not keys: - raise ValueError( - f"Dynamic {self.component.name} execution scope requires " - "component-grouped runtime patterns." - ) - return keys + @abstractmethod + def candidate_keys( + self, runtime_value: str | None + ) -> tuple[ComponentGroupKey, ...]: + """Return the keys one observed coordinate can execute under.""" + @abstractmethod def contains_runtime_key(self, key: Hashable | None) -> bool: """Return whether one concrete invocation belongs to this scope.""" - normalized_key = self.normalize_key(key) - if self.is_ungrouped: - return normalized_key is None - if self.is_dynamic: - return normalized_key is not None - return normalized_key in self.keys - + @abstractmethod def contains_scope(self, required: "ComponentGroupScope") -> bool: """Return whether every invocation in ``required`` is available here.""" - if self.is_ungrouped or required.is_ungrouped: - return self.is_ungrouped and required.is_ungrouped - if self.component is not required.component: - return False - if self.is_dynamic: - return True - if required.is_dynamic: - return False - return set(required.keys).issubset(self.keys) + def lies_within_keys(self, keys: tuple[str, ...]) -> bool: + """Whether every key of this scope is one of ``keys``.""" + return False + @abstractmethod def resolve_runtime_key( self, runtime_key: Hashable | None, ) -> ComponentGroupKey: """Resolve one concrete key against this compiler-owned scope.""" - if self.is_ungrouped: - return None - normalized_key = self.normalize_key(runtime_key) - if self.is_dynamic: - if normalized_key is None: - raise ValueError( - f"Dynamic {self.component.name} component scope requires a " - "concrete runtime key." - ) - return normalized_key - if normalized_key in self.keys: - return normalized_key - raise ValueError( - f"Component scope {self.component.name!r} groups {self.keys!r} do not " - f"contain runtime key {normalized_key!r}." - ) - def select_runtime_key( self, runtime_key: Hashable | None, ) -> ComponentGroupKey: """Select one coordinate from this compiled scope for an invocation.""" - - if self.is_ungrouped: - return None - if self.has_single_static_key: - return self.require_single_static_key() return self.resolve_runtime_key(runtime_key) - def require_single_static_key(self) -> str: - """Return the sole coordinate of an exact compiler-owned scope.""" + def selectable_from(self, producer_scope: "ComponentGroupScope") -> bool: + """Whether a consumer may select this scope from ``producer_scope``.""" + return producer_scope.contains_scope(self) - if not self.has_single_static_key: - raise ValueError( - "A single static component key is required, got " - f"component={self.component!r}, keys={self.keys!r}." - ) - key = self.keys[0] - if key is None: - raise RuntimeError("Static component scopes cannot contain None.") - return key + def owned_by(self, invocation_scope: "ComponentGroupScope") -> bool: + """Whether ``invocation_scope`` supplies the keys this scope selects.""" + return True + + def may_supply_key(self, component: type[Axis] | None, key: str) -> bool: + """Whether a source of this scope can feed group ``key`` of ``component``.""" + return True + + def shared_invocation_scope(self) -> "ComponentGroupScope": + """Scope of one callable applied across every group of this scope.""" + return self + + def excludes(self, other: "ComponentGroupScope") -> bool: + """Whether two compiled scopes of one component share no coordinate.""" + return False + + def keys_excluded_by(self, keys: tuple[str, ...]) -> bool: + return False + + def admits_record_key( + self, + execution_component: type[Axis] | None, + component_key: ComponentGroupKey, + record_key: str | None, + ) -> bool: + """Whether a producer record under ``record_key`` serves ``component_key``.""" + return True + + def value_for_plane( + self, + metadata: Mapping[str, object] | None, + component: type[Axis], + ) -> str | None: + """Return this scope's coordinate for one projected stack plane.""" + return self.select_runtime_key(None) def output_lineage_scope( self, @@ -235,10 +227,61 @@ def output_lineage_scope( consumer_variable_components: ComponentSet, ) -> "ComponentGroupScope": """Propagate this source scope through one consumer invocation.""" + return consumer_scope + + +@dataclass(frozen=True, slots=True) +class UngroupedScope(ComponentGroupScope): + """One default group: the invocation is not split by any component.""" + + component: ClassVar[None] = None + keys: ClassVar[tuple[None]] = (None,) + + def as_grouped(self) -> tuple["GroupedScope", ...]: + return () + + def runtime_keys(self, discovered_keys): + return (None,) + + def candidate_keys(self, runtime_value): + return (None,) + + def contains_runtime_key(self, key): + return self.normalize_key(key) is None + + def contains_scope(self, required): + return required == self + + def resolve_runtime_key(self, runtime_key): + return None + + def select_runtime_key(self, runtime_key): + return None + + +@dataclass(frozen=True, slots=True) +class GroupedScope(ComponentGroupScope): + """Groups split along one declared component axis.""" + + component: type[Axis] + + def __post_init__(self) -> None: + if not is_axis(self.component): + raise TypeError("ComponentGroupScope.component must be a declared axis.") + + def as_grouped(self) -> tuple["GroupedScope", ...]: + return (self,) + + def require_grouped(self, owner: str) -> "GroupedScope": + return self - if self.is_ungrouped or self.component in consumer_variable_components: + def groups_by(self, component: type[Axis] | None) -> bool: + return self.component is component + + def output_lineage_scope(self, consumer_scope, consumer_variable_components): + if self.component in consumer_variable_components: return consumer_scope - if consumer_scope.is_ungrouped or self.component is consumer_scope.component: + if consumer_scope.groups_by(self.component) or not consumer_scope.as_grouped(): return self raise ValueError( f"Output lineage component {self.component.name!r} is neither the " @@ -246,6 +289,153 @@ def output_lineage_scope( "of its variable components." ) + def contains_scope(self, required): + return required.groups_by(self.component) and self._contains_keys_of(required) + + def may_supply_key(self, component, key): + return self.component is not component or self.contains_runtime_key(key) + + @abstractmethod + def _contains_keys_of(self, required: ComponentGroupScope) -> bool: + """Whether every key of ``required`` (same component) is planned here.""" + + +@dataclass(frozen=True, slots=True) +class DynamicGroupScope(GroupedScope): + """Groups of one component whose keys are discovered at run time.""" + + keys: ClassVar[tuple[None]] = (None,) + has_compiled_keys = False + + def runtime_keys(self, discovered_keys): + keys = tuple( + dict.fromkeys( + self.normalize_key(key) for key in discovered_keys if key is not None + ) + ) + if not keys: + raise ValueError( + f"Dynamic {self.component.name} execution scope requires " + "component-grouped runtime patterns." + ) + return keys + + def candidate_keys(self, runtime_value): + return () if runtime_value is None else (self.normalize_key(runtime_value),) + + def contains_runtime_key(self, key): + return self.normalize_key(key) is not None + + def _contains_keys_of(self, required): + return True + + def resolve_runtime_key(self, runtime_key): + normalized_key = self.normalize_key(runtime_key) + if normalized_key is None: + raise ValueError( + f"Dynamic {self.component.name} component scope requires a " + "concrete runtime key." + ) + return normalized_key + + def selectable_from(self, producer_scope): + return producer_scope.groups_by(self.component) + + def owned_by(self, invocation_scope): + return invocation_scope.groups_by(self.component) + + def admits_record_key(self, execution_component, component_key, record_key): + return self.component is not execution_component or record_key == component_key + + def value_for_plane(self, metadata, component): + from openhcs.core.source_matching import semantic_source_metadata_value + + value = semantic_source_metadata_value(metadata, component.name) + return None if value is None else str(value) + + +@dataclass(frozen=True, slots=True) +class StaticGroupScope(GroupedScope): + """Several groups of one component named at compile time.""" + + keys: tuple[str, ...] + names_several_keys = True + + def __post_init__(self) -> None: + GroupedScope.__post_init__(self) + if len(self.keys) < self.minimum_key_count() or any( + not isinstance(key, str) for key in self.keys + ): + raise ValueError( + f"{type(self).__name__} keys must be at least " + f"{self.minimum_key_count()} strings, got {self.keys!r}." + ) + + @staticmethod + def minimum_key_count() -> int: + return 2 + + def runtime_keys(self, discovered_keys): + return self.keys + + def candidate_keys(self, runtime_value): + return self.keys + + def contains_runtime_key(self, key): + return self.normalize_key(key) in self.keys + + def _contains_keys_of(self, required): + return required.lies_within_keys(self.keys) + + def lies_within_keys(self, keys): + return set(self.keys).issubset(keys) + + def resolve_runtime_key(self, runtime_key): + normalized_key = self.normalize_key(runtime_key) + if normalized_key in self.keys: + return normalized_key + raise ValueError( + f"Component scope {self.component.name!r} groups {self.keys!r} do not " + f"contain runtime key {normalized_key!r}." + ) + + def shared_invocation_scope(self): + return DynamicGroupScope(self.component) + + def excludes(self, other): + return other.keys_excluded_by(self.keys) + + def keys_excluded_by(self, keys): + return set(self.keys).isdisjoint(keys) + + +@dataclass(frozen=True, slots=True, init=False) +class SingleKeyScope(StaticGroupScope): + """Exactly one group of one component, named at compile time.""" + + names_several_keys = False + + def __init__(self, component: type[Axis], key: str) -> None: + object.__setattr__(self, "component", component) + object.__setattr__(self, "keys", (key,)) + self.__post_init__() + + @staticmethod + def minimum_key_count() -> int: + return 1 + + def single_key(self) -> str: + return self.keys[0] + + def require_single_key(self) -> str: + return self.keys[0] + + def select_runtime_key(self, runtime_key): + return self.keys[0] + + def shared_invocation_scope(self): + return self + @dataclass(frozen=True, slots=True) class RuntimeExecutionAxisScope: @@ -480,6 +670,16 @@ def coordinate_label(self) -> str: for component, value in self.presentation_component_values ) + def for_output_scope( + self, output_scope: ComponentGroupScope + ) -> "RuntimeExecutionAxisScope": + """Project this execution identity onto an output's single group key.""" + + key = output_scope.single_key() + return self.for_group_coordinate( + None if key is None else output_scope.component, key + ) + def for_group_coordinate( self, component: type[Axis] | None, @@ -567,11 +767,8 @@ def has_value(self) -> bool: def group_scope(self) -> ComponentGroupScope: """Return this runtime coordinate as one concrete component-group scope.""" if self.component is None: - return ComponentGroupScope.ungrouped() - return ComponentGroupScope( - (self.require_value_text(),), - component=self.component, - ) + return UngroupedScope() + return SingleKeyScope(self.component, self.require_value_text()) @property def cache_key(self) -> tuple[tuple[str, str], ...]: diff --git a/openhcs/core/debug.py b/openhcs/core/debug.py index 320a243bd0..60c7a18433 100644 --- a/openhcs/core/debug.py +++ b/openhcs/core/debug.py @@ -395,7 +395,7 @@ def from_artifact_plan(cls, plan: ArtifactPlan) -> "DebugArtifactIdentity": return cls( name=plan.name, kind=plan.artifact_type, - group_keys=tuple(plan.group_keys or (None,)), + group_keys=plan.group_scope.keys, producer_step_index=( None if output_plan is None or output_plan.producer_step_index is None diff --git a/openhcs/core/pipeline/compiler.py b/openhcs/core/pipeline/compiler.py index 9dfa0fe33d..1620eade26 100644 --- a/openhcs/core/pipeline/compiler.py +++ b/openhcs/core/pipeline/compiler.py @@ -1210,7 +1210,7 @@ def _validate_payload_axis_source_anchor( """Validate an exact source anchor, or leave ancestry work outstanding.""" declared_plan = plan.source_binding_plan - if grouped_pattern and plan.execution_group_scope.component is None: + if grouped_pattern and not plan.execution_group_scope.as_grouped(): raise ValueError( f"{owner}: cannot project source bindings for a grouped callable " "pattern without a routed component scope." @@ -1220,16 +1220,13 @@ def _validate_payload_axis_source_anchor( try: if not declared_plan.has_primary_content: component_value = None - elif execution_scope.component is None: - component_value = None - elif not grouped_pattern and execution_scope.has_single_static_key: - component_value = execution_scope.require_single_static_key() elif not grouped_pattern: - # A single callable is applied across every static/dynamic routed - # coordinate. The compiled plan itself is the exact source scope; - # it must be validated as a whole rather than guessed from the - # synthetic ``default`` function-group key. - component_value = None + # A single callable over one exact coordinate projects it. Over + # several static/dynamic routed coordinates (or none), the + # compiled plan itself is the exact source scope; it is + # validated as a whole rather than guessed from the synthetic + # ``default`` function-group key. + component_value = execution_scope.single_key() else: component_value = execution_scope.resolve_runtime_key(group.group_key) if declared_plan.has_primary_content: diff --git a/openhcs/core/pipeline/path_planner.py b/openhcs/core/pipeline/path_planner.py index 89ddfcaf44..3b0be76fd5 100644 --- a/openhcs/core/pipeline/path_planner.py +++ b/openhcs/core/pipeline/path_planner.py @@ -46,7 +46,12 @@ InputConversionPlan, MaterializedOutputPlan, ) -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + DynamicGroupScope, + SingleKeyScope, + UngroupedScope, +) from openhcs.core.component_set import ComponentSet from openhcs.core.config import ( PathPlanningConfig, @@ -104,90 +109,11 @@ def __init__( ) -class PathPlannerGroupScope(ComponentGroupScope): - """Component-group scope with artifact-planner projections.""" - - @classmethod - def from_output_plan( - cls, - output_plan: ArtifactOutputPlan, - ) -> "PathPlannerGroupScope": - if output_plan.group_keys: - return cls.from_raw( - output_plan.group_keys, - component=output_plan.group_component, - ) - return cls.ungrouped() - - @classmethod - def from_plan( - cls, - plan: ArtifactInputPlan | ArtifactOutputPlan, - ) -> "PathPlannerGroupScope": - if plan.group_keys: - return cls.from_raw( - plan.group_keys, - component=plan.group_component, - ) - return cls.ungrouped() - - @classmethod - def relation_scope_from_plan( - cls, - plan: ArtifactInputPlan | ArtifactOutputPlan, - component: type[Axis] | None, - ) -> "PathPlannerGroupScope": - """Return the plan domain relevant to one relation component.""" - - domain = None if component is None else plan.component_domain(component) - if domain is None: - return cls.from_plan(plan) - return cls.from_raw(domain.keys, component=domain.component) - - def output_groups_for( - self, - output_refs: Iterable[ArtifactSpecRef], - ) -> dict[ArtifactSpecRef, tuple[PlannerGroupKey, ...]]: - return {output_ref: self.keys for output_ref in output_refs} - - @classmethod - def union_compatible( - cls, - scopes: Sequence["PathPlannerGroupScope"], - ) -> "PathPlannerGroupScope | None": - """Union scopes that describe the same grouped component.""" - - if not scopes: - return None - component = scopes[0].component - if any(scope.component != component for scope in scopes[1:]): - return None - if any(scope.is_dynamic for scope in scopes): - if component is None: - raise RuntimeError( - "Dynamic component scope lost its component identity." - ) - return cls.dynamic(component) - return cls.from_raw( - dict.fromkeys(key for scope in scopes for key in scope.keys), - component=component, - ) - - def missing_from( - self, - producer_scope: "PathPlannerGroupScope", - ) -> list[PlannerGroupKey]: - return [group for group in self.keys if group not in producer_scope.keys] - - def single_group_key(self) -> PlannerGroupKey | None: - return self.require_single_static_key() if self.has_single_static_key else None - - @dataclass(frozen=True) class PathPlannerComponentScopes: """Component value scopes carried by the main-flow image branch.""" - scopes: Mapping[type[Axis], PathPlannerGroupScope] + scopes: Mapping[type[Axis], ComponentGroupScope] @classmethod def empty(cls) -> "PathPlannerComponentScopes": @@ -196,16 +122,16 @@ def empty(cls) -> "PathPlannerComponentScopes": def scope_for_group_by( self, group_by: type[GroupingDeclaration] | None, - ) -> PathPlannerGroupScope | None: + ) -> ComponentGroupScope | None: group_by_component = self.component_from_group_by(group_by) if group_by_component is None: - return PathPlannerGroupScope.ungrouped() + return UngroupedScope() return self.scopes.get(group_by_component) def output_after( self, step: AbstractStep, - execution_scope: PathPlannerGroupScope, + execution_scope: ComponentGroupScope, compiled_pattern: CompiledFunctionPattern | None, ) -> "PathPlannerComponentScopes": if not isinstance(step, FunctionStep): @@ -223,14 +149,14 @@ def output_after( for component in variable_components: scopes.pop(component, None) if compiled_pattern.collapses_input_plane_axis(): - scopes[component] = PathPlannerGroupScope.ungrouped() + scopes[component] = UngroupedScope() group_by = PathPlannerExecutionGroups.normalized_group_by(step) group_by_component = self.component_from_group_by(group_by) if ( group_by_component is not None and group_by_component not in variable_components - and not execution_scope.is_ungrouped + and execution_scope.as_grouped() ): scopes[group_by_component] = execution_scope @@ -251,10 +177,10 @@ class ArtifactPlanMaps: """Compiled artifact I/O maps for one step.""" declarations: ArtifactGraph - group_scope: PathPlannerGroupScope + group_scope: ComponentGroupScope inputs: dict[ArtifactSpecRef, ArtifactInputPlan] outputs: dict[ArtifactSpecRef, ArtifactOutputPlan] - relation_source_scopes: Mapping[ArtifactSpecRef, PathPlannerGroupScope] + relation_source_scopes: Mapping[ArtifactSpecRef, ComponentGroupScope] source_binding_plan: CompiledSourceBindingPlan source_universe_plan: CompiledSourceUniversePlan @@ -267,40 +193,38 @@ class PathPlannerExecutionGroups: @staticmethod def normalize_group_key(key: Hashable | None) -> PlannerGroupKey: - return PathPlannerGroupScope.normalize_key(key) + return ComponentGroupScope.normalize_key(key) def context_producer_scope( self, producer: ArtifactProducer, *, source_component: type[Axis] | None, - ) -> PathPlannerGroupScope: + ) -> ComponentGroupScope: """Resolve producer groups against their own compiled axis authority.""" output_plan = self.planner.declared.get( producer.spec.ref().for_plan_type(ArtifactOutputPlan) ) if output_plan is not None: - return PathPlannerGroupScope.relation_scope_from_plan( - output_plan, source_component - ) + return output_plan.relation_scope(source_component) if producer.producer_step_index is not None: scope = self.planner.plans[producer.producer_step_index].execution_group_scope - return PathPlannerGroupScope.from_raw(scope.keys, component=scope.component) + return scope if producer.spec.ref() != PIPELINE_INPUT_ARTIFACT.ref(): - return PathPlannerGroupScope.from_raw( + return ComponentGroupScope.of( producer.groups, component=source_component ) if source_component is None: - return PathPlannerGroupScope.ungrouped() + return UngroupedScope() source_keys = tuple( self.planner.orchestrator.get_component_keys( source_component, resolved_config=self.planner.session.global_config ) ) return ( - PathPlannerGroupScope.from_raw(source_keys, component=source_component) - if source_keys else PathPlannerGroupScope.dynamic(source_component) + ComponentGroupScope.of(source_keys, component=source_component) + if source_keys else DynamicGroupScope(source_component) ) def get_execution_groups( @@ -310,15 +234,15 @@ def get_execution_groups( *, source_bindings: SourceBindingDeclarationsMixin | None = None, contracts: Sequence[CallableContract] = (), - ) -> PathPlannerGroupScope: + ) -> ComponentGroupScope: """Determine which component groups this step will execute for.""" if not isinstance(step, FunctionStep): - return PathPlannerGroupScope.ungrouped() + return UngroupedScope() func_pattern = normalize_function_pattern(step.func) group_by = self.normalized_group_by(step) if func_pattern.is_grouped: - scope = PathPlannerGroupScope.from_raw( + scope = ComponentGroupScope.of( func_pattern.source_group_keys, component=self.execution_component_for_dict_pattern( group_by, @@ -354,9 +278,9 @@ def get_execution_groups( source_bindings=source_bindings, ) scope = ( - self.dynamic_execution_scope_for_group_by(step, group_by) - if source_scope.is_ungrouped - else source_scope + source_scope + if source_scope.as_grouped() + else self.dynamic_execution_scope_for_group_by(step, group_by) ) if artifact_owned: scope = self.artifact_owned_execution_scope( @@ -378,8 +302,8 @@ def artifact_owned_execution_scope( step: AbstractStep, contracts: Sequence[CallableContract], *, - consumer_scope: PathPlannerGroupScope, - ) -> PathPlannerGroupScope: + consumer_scope: ComponentGroupScope, + ) -> ComponentGroupScope: """Resolve invocation groups from their declared input-domain owners.""" owner_specs = tuple( @@ -404,16 +328,13 @@ def artifact_owned_execution_scope( if group_component is None else AxisFamily.active().require(group_component) ) - scopes: list[PathPlannerGroupScope] = [] + scopes: list[ComponentGroupScope] = [] for spec in owner_specs: producer_ref = spec.ref().for_plan_type(ArtifactOutputPlan) producer = self.planner.declared.get(producer_ref) if producer is not None: scopes.append( - PathPlannerGroupScope.relation_scope_from_plan( - producer, - normalized_group_component, - ) + producer.relation_scope(normalized_group_component) ) continue @@ -447,7 +368,7 @@ def artifact_owned_execution_scope( f"Artifact-owned FunctionStep {step.name!r} cannot " f"resolve group scope for {spec.ref()!r}." ) - scopes.append(PathPlannerGroupScope.ungrouped()) + scopes.append(UngroupedScope()) continue source_group_keys = ( @@ -462,20 +383,19 @@ def artifact_owned_execution_scope( ), ) ) - source_scope = ( - PathPlannerGroupScope.from_raw( + if not source_group_keys and group_by is not Ungrouped: + raise ValueError( + f"Artifact-owned FunctionStep {step.name!r} cannot " + f"resolve group scope for {spec.ref()!r}." + ) + scopes.append( + ComponentGroupScope.of( source_group_keys, component=normalized_group_component, ) if source_group_keys - else PathPlannerGroupScope.ungrouped() + else UngroupedScope() ) - if source_scope.is_ungrouped and group_by is not Ungrouped: - raise ValueError( - f"Artifact-owned FunctionStep {step.name!r} cannot " - f"resolve group scope for {spec.ref()!r}." - ) - scopes.append(source_scope) if not scopes: return consumer_scope @@ -490,7 +410,7 @@ def artifact_owned_execution_scope( ) for scope in scopes ) - execution_scope = PathPlannerGroupScope.union_compatible(projected_scopes) + execution_scope = ComponentGroupScope.union_compatible(projected_scopes) if execution_scope is None: raise ValueError( f"Artifact-owned FunctionStep {step.name!r} has " @@ -502,11 +422,11 @@ def dynamic_execution_scope_for_group_by( self, step: AbstractStep, group_by: type[GroupingDeclaration] | None, - ) -> PathPlannerGroupScope: + ) -> ComponentGroupScope: """Return a typed runtime-discovered scope for a concrete group axis.""" group_by_component = PathPlannerComponentScopes.component_from_group_by(group_by) if group_by_component is None: - return PathPlannerGroupScope.ungrouped() + return UngroupedScope() source_keys = ( tuple( self.planner.orchestrator.get_component_keys( @@ -518,11 +438,11 @@ def dynamic_execution_scope_for_group_by( else () ) if source_keys: - return PathPlannerGroupScope.from_raw( + return ComponentGroupScope.of( source_keys, component=AxisFamily.active().require(group_by_component), ) - return PathPlannerGroupScope.dynamic( + return DynamicGroupScope( AxisFamily.active().require(group_by_component) ) @@ -532,16 +452,16 @@ def source_binding_scope_for_group_by( group_by: type[GroupingDeclaration] | None, *, source_bindings: SourceBindingDeclarationsMixin | None = None, - ) -> PathPlannerGroupScope: + ) -> ComponentGroupScope: """Derive execution groups declared by source-binding component identity.""" group_by_component = PathPlannerComponentScopes.component_from_group_by(group_by) if group_by_component is None: - return PathPlannerGroupScope.ungrouped() + return UngroupedScope() if source_bindings is None: source_bindings = step.source_bindings if not source_bindings.binding_declarations: - return PathPlannerGroupScope.ungrouped() + return UngroupedScope() component = AxisFamily.active().require(group_by_component) group_keys = source_binding_group_keys_for_group_by( source_bindings, @@ -549,8 +469,8 @@ def source_binding_scope_for_group_by( realized_source_metadata=self.planner.session.realized_source_metadata, ) if not group_keys: - return PathPlannerGroupScope.ungrouped() - return PathPlannerGroupScope.from_raw(group_keys, component=component) + return UngroupedScope() + return ComponentGroupScope.of(group_keys, component=component) @staticmethod def execution_component_for_dict_pattern( @@ -620,7 +540,7 @@ def source_binding_component_domains( specs: Iterable[ArtifactSpec], source_bindings: SourceBindingDeclarationsMixin, available_artifacts: ArtifactSpecCollection, - ) -> tuple[PathPlannerGroupScope, ...]: + ) -> tuple[ComponentGroupScope, ...]: """Compile every declared component domain owned by source bindings.""" bindings = source_bindings.bindings_for_artifact_specs( @@ -634,7 +554,7 @@ def source_binding_component_domains( for binding in bindings ) return tuple( - PathPlannerGroupScope.from_raw(values, component=component) + ComponentGroupScope.of(values, component=component) for component in AxisFamily.active().axes for values in ( tuple(dict.fromkeys( @@ -651,7 +571,7 @@ def compile_plan_maps( step: AbstractStep, step_index: int, declarations: ArtifactGraph, - group_scope: PathPlannerGroupScope, + group_scope: ComponentGroupScope, execution_scope: FunctionStepExecutionScope = FunctionStepExecutionScope.AXIS, source_bindings: StepSourceBindingsConfig | CompiledSourceBindingPlan = EMPTY_SOURCE_BINDINGS, ) -> ArtifactPlanMaps: @@ -748,18 +668,18 @@ def output_groups_from_declared_relations( self, declarations: ArtifactGraph, *, - group_scope: PathPlannerGroupScope, - relation_source_scopes: Mapping[ArtifactSpecRef, PathPlannerGroupScope], + group_scope: ComponentGroupScope, + relation_source_scopes: Mapping[ArtifactSpecRef, ComponentGroupScope], consumer_variable_components: ComponentSet, step_index: int, step_name: str | None, - ) -> Mapping[ArtifactSpecRef, PathPlannerGroupScope]: + ) -> Mapping[ArtifactSpecRef, ComponentGroupScope]: """Return output groups after applying declared artifact relations.""" - output_groups: dict[ArtifactSpecRef, PathPlannerGroupScope] = {} + output_groups: dict[ArtifactSpecRef, ComponentGroupScope] = {} for producer in declarations.producers: output_ref = producer.spec.ref() if producer.has_explicit_invocation_group_ownership(): - output_groups[output_ref] = PathPlannerGroupScope.from_raw( + output_groups[output_ref] = ComponentGroupScope.of( producer.groups, component=group_scope.component, ) @@ -801,14 +721,14 @@ def output_relation_source_scopes_by_ref( declarations: ArtifactGraph, relation_source_scopes: Mapping[ ArtifactSpecRef, - PathPlannerGroupScope, + ComponentGroupScope, ], - ) -> dict[ArtifactSpecRef, PathPlannerGroupScope]: + ) -> dict[ArtifactSpecRef, ComponentGroupScope]: """Project input-qualified lineage for outputs without changing storage scope.""" resolved = dict(relation_source_scopes) - def declared_scope(source_ref: ArtifactSpecRef) -> PathPlannerGroupScope: + def declared_scope(source_ref: ArtifactSpecRef) -> ComponentGroupScope: try: return relation_source_scopes[source_ref] except KeyError as exc: @@ -824,7 +744,7 @@ def declared_scope(source_ref: ArtifactSpecRef) -> PathPlannerGroupScope: declared_scope(source_ref) if ref == source_ref else resolved[ref] for ref in lineage_refs ) - projected = PathPlannerGroupScope.union_compatible(lineage_scopes) + projected = ComponentGroupScope.union_compatible(lineage_scopes) if projected is None: raise ValueError( f"Input {source_ref!r} has incompatible group-lineage scopes " @@ -836,8 +756,8 @@ def declared_scope(source_ref: ArtifactSpecRef) -> PathPlannerGroupScope: @staticmethod def output_group_scope_sources_by_group( spec: ArtifactSpec, - group_scope: PathPlannerGroupScope, - relation_source_scopes: Mapping[ArtifactSpecRef, PathPlannerGroupScope], + group_scope: ComponentGroupScope, + relation_source_scopes: Mapping[ArtifactSpecRef, ComponentGroupScope], ) -> dict[str | None, tuple[ArtifactSpecRef, ...]]: """Compile exact relation sources for each output component group.""" @@ -846,18 +766,14 @@ def output_group_scope_sources_by_group( return {} sources_by_group: dict[str | None, tuple[ArtifactSpecRef, ...]] = {} for group_key in group_scope.keys: - if group_scope.is_ungrouped or group_key is None: + if group_key is None: selected = source_refs else: selected = tuple( source_ref for source_ref in source_refs - for source_scope in (relation_source_scopes[source_ref],) - if ( - source_scope.is_ungrouped - or source_scope.component is not group_scope.component - or source_scope.is_dynamic - or source_scope.contains_runtime_key(group_key) + if relation_source_scopes[source_ref].may_supply_key( + group_scope.component, group_key ) ) if not selected: @@ -873,13 +789,13 @@ def relation_source_scopes_by_ref( declarations: ArtifactGraph, artifact_inputs: Mapping[ArtifactSpecRef, ArtifactInputPlan], *, - group_scope: PathPlannerGroupScope, + group_scope: ComponentGroupScope, source_bindings: SourceBindingDeclarationsMixin, group_by: type[GroupingDeclaration] | None, - ) -> dict[ArtifactSpecRef, PathPlannerGroupScope]: + ) -> dict[ArtifactSpecRef, ComponentGroupScope]: """Return exact group scopes for every declared relation source.""" - source_scopes_by_ref: dict[ArtifactSpecRef, PathPlannerGroupScope] = {} + source_scopes_by_ref: dict[ArtifactSpecRef, ComponentGroupScope] = {} ArtifactInputPlan.require_exact_map( artifact_inputs, boundary="Path planner artifact input", @@ -906,10 +822,7 @@ def relation_source_scopes_by_ref( ) continue source_scopes_by_ref[spec.ref()] = ( - PathPlannerGroupScope.relation_scope_from_plan( - input_plan, - component, - ) + input_plan.relation_scope(component) ) for binding in source_bindings.binding_declarations: identity_values = ( @@ -923,7 +836,7 @@ def relation_source_scopes_by_ref( else () ) binding_scope = ( - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( identity_values, component=component, ) @@ -940,10 +853,7 @@ def relation_source_scopes_by_ref( source_scopes_by_ref.setdefault( consumer.spec.ref(), ( - PathPlannerGroupScope.relation_scope_from_plan( - producer_plan, - component, - ) + producer_plan.relation_scope(component) if producer_plan is not None else group_scope ), @@ -967,10 +877,7 @@ def relation_source_scopes_by_ref( if producer_plan is None: continue source_scopes_by_ref[source_ref] = ( - PathPlannerGroupScope.relation_scope_from_plan( - producer_plan, - component, - ) + producer_plan.relation_scope(component) ) for spec in dict.fromkeys( consumer.spec @@ -989,7 +896,7 @@ def relation_source_scopes_by_ref( if source_ref in source_scopes_by_ref and source_scopes_by_ref[source_ref].component is group_scope.component ) - inherited_scope = PathPlannerGroupScope.union_compatible( + inherited_scope = ComponentGroupScope.union_compatible( source_stack_scopes ) if inherited_scope is not None: @@ -1004,9 +911,9 @@ def build_step_compiled_function_pattern( artifact_outputs: Mapping[ArtifactSpecRef, ArtifactOutputPlan], relation_source_scopes: Mapping[ ArtifactSpecRef, - PathPlannerGroupScope, + ComponentGroupScope, ], - execution_group_scope: PathPlannerGroupScope, + execution_group_scope: ComponentGroupScope, declarations: ArtifactGraph, ) -> CompiledFunctionPattern: """Build the executable function-pattern graph for a FunctionStep.""" @@ -1068,9 +975,9 @@ def compile_invocation_input_edges( artifact_inputs: Mapping[ArtifactSpecRef, ArtifactInputPlan], relation_source_scopes: Mapping[ ArtifactSpecRef, - PathPlannerGroupScope, + ComponentGroupScope, ], - execution_group_scope: PathPlannerGroupScope, + execution_group_scope: ComponentGroupScope, consumer_variable_components: ComponentSet, source_bindings: StepSourceBindingsConfig | CompiledSourceBindingPlan = EMPTY_SOURCE_BINDINGS, available_artifacts: ArtifactSpecCollection = ArtifactSpecCollection(()), @@ -1084,27 +991,15 @@ def compile_invocation_input_edges( invocations: list[CompiledFunctionInvocation] = [] for invocation in group.invocations: if compiled_pattern.is_grouped: - if execution_group_scope.is_ungrouped: - raise ValueError( - f"Grouped invocation {invocation.key!r} has no grouped " - "execution component." - ) - invocation_scope = PathPlannerGroupScope.from_raw( - ( - execution_group_scope.resolve_runtime_key( - invocation.key.group_key - ), - ), - component=execution_group_scope.component, + grouped_scope = execution_group_scope.require_grouped( + f"Grouped invocation {invocation.key!r} execution" ) - elif execution_group_scope.is_ungrouped: - invocation_scope = execution_group_scope - elif execution_group_scope.has_single_static_key: - invocation_scope = execution_group_scope - else: - invocation_scope = PathPlannerGroupScope.dynamic( - execution_group_scope.component + invocation_scope = SingleKeyScope( + grouped_scope.component, + grouped_scope.resolve_runtime_key(invocation.key.group_key), ) + else: + invocation_scope = execution_group_scope.shared_invocation_scope() selected_plans = invocation.contract.select_plans( ArtifactInputPlan, artifact_inputs, @@ -1175,10 +1070,10 @@ def invocation_input_edge( *, input_spec: ArtifactSpec, storage_plan: ArtifactInputPlan, - invocation_scope: PathPlannerGroupScope, + invocation_scope: ComponentGroupScope, relation_source_scopes: Mapping[ ArtifactSpecRef, - PathPlannerGroupScope, + ComponentGroupScope, ], consumer_variable_components: ComponentSet, source_bindings: SourceBindingDeclarationsMixin, @@ -1201,7 +1096,7 @@ def invocation_input_edge( if invocation.contract.execution_scope is FunctionStepExecutionScope.PLATE: projection = ArtifactInputProjectionPlan( invocation_scope=invocation_scope, - producer_selection_scope=PathPlannerGroupScope.from_plan(storage_plan), + producer_selection_scope=storage_plan.group_scope, ) return InvocationArtifactInputEdgePlan( key=input_edge_key, @@ -1229,10 +1124,7 @@ def invocation_input_edge( ) ) storage_domains = tuple( - PathPlannerGroupScope.from_raw( - domain.keys, - component=domain.component, - ) + domain for domain in storage_plan.component_domains ) component_scopes = self.exact_component_scopes( @@ -1274,10 +1166,10 @@ def require_relation_source_scope( source_ref: ArtifactSpecRef, relation_source_scopes: Mapping[ ArtifactSpecRef, - PathPlannerGroupScope, + ComponentGroupScope, ], invocation: CompiledFunctionInvocation, - ) -> PathPlannerGroupScope: + ) -> ComponentGroupScope: """Return one exact compiler-resolved relation source scope.""" try: @@ -1303,20 +1195,15 @@ def exact_component_scopes( by_component: dict[type[Axis], ComponentGroupScope] = {} storage_components: set[type[Axis]] = set() for domain in storage_domains: - if domain.is_ungrouped or domain.is_dynamic or len(domain.keys) != 1: + if domain.single_key() is None: continue - component = domain.component - if component is None: - raise RuntimeError("Grouped storage domain lost its component.") - by_component[component] = domain - storage_components.add(component) + by_component[domain.component] = domain + storage_components.add(domain.component) - for scope in relation_scopes: - if scope.is_ungrouped: - continue + for scope in ( + grouped for scope in relation_scopes for grouped in scope.as_grouped() + ): component = scope.component - if component is None: - raise RuntimeError("Grouped projection scope lost its component.") existing = by_component.get(component) if component in storage_components: continue @@ -1328,43 +1215,34 @@ def exact_component_scopes( ) by_component[component] = scope - for domain in component_domains: - if domain.is_ungrouped: - continue + for domain in ( + grouped for domain in component_domains for grouped in domain.as_grouped() + ): component = domain.component - if component is None: - raise RuntimeError("Grouped component domain lost its component.") exact_scope = by_component.get(component) - if ( - exact_scope is not None - and not exact_scope.is_dynamic - and not domain.contains_scope(exact_scope) - ): + if exact_scope is not None and not exact_scope.selectable_from(domain): raise ValueError( f"Invocation {invocation.key!r} input {artifact_ref!r} " f"has exact {component.name!r} coordinate {exact_scope!r} " f"outside component domain {domain!r}." ) - if not invocation_scope.is_ungrouped: - component = invocation_scope.component - if component is None: - raise RuntimeError("Grouped invocation scope lost its component.") - by_component.setdefault(component, invocation_scope) + for grouped in invocation_scope.as_grouped(): + by_component.setdefault(grouped.component, grouped) return tuple(by_component.values()) def process_artifact_outputs( self, declarations: ArtifactGraph, sid: int, - output_groups: Mapping[ArtifactSpecRef, PathPlannerGroupScope] | None = None, + output_groups: Mapping[ArtifactSpecRef, ComponentGroupScope] | None = None, *, execution_scope: FunctionStepExecutionScope, artifact_inputs: Mapping[ArtifactSpecRef, ArtifactInputPlan], relation_source_scopes: ( Mapping[ ArtifactSpecRef, - PathPlannerGroupScope, + ComponentGroupScope, ] | None ) = None, @@ -1386,10 +1264,10 @@ def process_artifact_outputs( "Path planner output-group maps require ArtifactSpecRef " f"keys, got {type(output_ref).__name__}." ) - if not isinstance(output_scope, PathPlannerGroupScope): + if not isinstance(output_scope, ComponentGroupScope): raise TypeError( "Path planner output-group maps require " - "PathPlannerGroupScope values, got " + "ComponentGroupScope values, got " f"{type(output_scope).__name__} for {output_ref!r}." ) if output_ref not in outputs: @@ -1417,23 +1295,20 @@ def process_artifact_outputs( group_scope = ( output_groups[output_ref] if output_groups is not None and output_ref in output_groups - else PathPlannerGroupScope.ungrouped() + else UngroupedScope() ) paths_by_group = self.planner.paths.paths_by_group(str(path), group_scope) source_stack_axes: list[ComponentSet] = [] - source_stack_domains: list[PathPlannerGroupScope] = [] - source_binding_domains: list[PathPlannerGroupScope] = [] + source_stack_domains: list[ComponentGroupScope] = [] + source_binding_domains: list[ComponentGroupScope] = [] for source_ref in spec.source_stack_scope_sources(): source_plan = artifact_inputs.get(source_ref) relation_scope = relation_source_scopes.get(source_ref) - if relation_scope is not None and not relation_scope.is_ungrouped: - source_stack_domains.append(relation_scope) + if relation_scope is not None: + source_stack_domains.extend(relation_scope.as_grouped()) if source_plan is not None: source_stack_domains.extend( - PathPlannerGroupScope.from_raw( - domain.keys, - component=domain.component, - ) + domain for domain in source_plan.component_domains ) source_stack_axes.append( @@ -1481,43 +1356,34 @@ def process_artifact_outputs( ) lineage_domains_by_component: dict[ type[Axis], - list[PathPlannerGroupScope], + list[ComponentGroupScope], ] = defaultdict(list) for domain in source_stack_domains: - if domain.component is None: - raise RuntimeError( - "Artifact output source-stack domain lost its component." - ) if domain not in lineage_domains_by_component[domain.component]: lineage_domains_by_component[domain.component].append(domain) binding_domains_by_component: dict[ type[Axis], - list[PathPlannerGroupScope], + list[ComponentGroupScope], ] = defaultdict(list) for domain in source_binding_domains: - if domain.component is None: - raise RuntimeError( - "Artifact output source-binding domain lost its component." - ) if domain not in binding_domains_by_component[domain.component]: binding_domains_by_component[domain.component].append(domain) - component_domains_list: list[PathPlannerGroupScope] = [] + component_domains_list: list[ComponentGroupScope] = [] components = ComponentSet.collect( lineage_domains_by_component, binding_domains_by_component, ) for component in components: - lineage_domain = PathPlannerGroupScope.union_compatible( + lineage_domain = ComponentGroupScope.union_compatible( lineage_domains_by_component[component] ) - binding_domain = PathPlannerGroupScope.union_compatible( + binding_domain = ComponentGroupScope.union_compatible( binding_domains_by_component[component] ) if ( lineage_domain is not None and binding_domain is not None - and not lineage_domain.is_dynamic - and not binding_domain.contains_scope(lineage_domain) + and not lineage_domain.selectable_from(binding_domain) ): raise ValueError( f"Artifact output {spec.ref()!r} inherits " @@ -1559,8 +1425,7 @@ def process_artifact_outputs( relation_source_scopes, ) ), - group_keys=group_scope.keys, - group_component=group_scope.component, + group_scope=group_scope, variable_components=( source_stack_axes[0].as_tuple() if source_stack_axes else () ), @@ -1578,7 +1443,7 @@ def process_artifact_inputs( self, declarations: ArtifactGraph, sid: int, - consumer_scope: PathPlannerGroupScope, + consumer_scope: ComponentGroupScope, source_bindings: SourceBindingDeclarationsMixin, variable_components: ComponentSet, step_name: Optional[str] = None, @@ -1627,8 +1492,7 @@ def process_artifact_inputs( artifact_type=input_spec.artifact_type, sidecar_role=input_spec.sidecar_role, relations=output_spec.relations, - group_keys=consumer_scope.keys, - group_component=consumer_scope.component, + group_scope=consumer_scope, variable_components=variable_components.as_tuple(), source_step_id=sid, source_step_scope_id=self.planner.plans[sid].step_scope_id, @@ -1676,7 +1540,7 @@ def _producer_artifact_input_plan( f"sidecar role {input_spec.sidecar_role!r}, but producer step " f"'{producer_name}' provides {producer.sidecar_role!r}." ) - producer_scope = PathPlannerGroupScope.from_output_plan(producer) + producer_scope = producer.group_scope producer_paths_by_group = self._producer_artifact_paths_by_group(producer) return ArtifactInputPlan( @@ -1690,8 +1554,7 @@ def _producer_artifact_input_plan( sidecar_role=input_spec.sidecar_role, relations=producer.relations, paths_by_group=producer_paths_by_group, - group_keys=producer_scope.keys, - group_component=producer_scope.component, + group_scope=producer_scope, variable_components=producer.variable_components, component_domains=producer.component_domains, source_step_id=producer.producer_step_index, @@ -1728,10 +1591,10 @@ def _producer_artifact_paths_by_group( @staticmethod def _producer_artifact_input_path( producer: ArtifactOutputPlan, - producer_scope: PathPlannerGroupScope, + producer_scope: ComponentGroupScope, paths_by_group: Mapping[str | None, str], ) -> str: - producer_group = producer_scope.single_group_key() + producer_group = producer_scope.single_key() if producer_group is not None: return paths_by_group.get(producer_group, producer.path) return producer.path @@ -2088,7 +1951,7 @@ def plan_step(self, step: AbstractStep, step_index: int) -> None: input_component_scopes = self.input_component_scopes(main_input_dependency) input_dir, output_dir = self.step_io_dirs(main_input_dependency, step_index) group_scope = ( - PathPlannerGroupScope.ungrouped() + UngroupedScope() if execution_scope is FunctionStepExecutionScope.PLATE else self.planner.execution_groups.get_execution_groups( step, diff --git a/openhcs/core/runtime_artifact_values.py b/openhcs/core/runtime_artifact_values.py index 4ff6e5dd2c..1a85cd56c2 100644 --- a/openhcs/core/runtime_artifact_values.py +++ b/openhcs/core/runtime_artifact_values.py @@ -10,7 +10,7 @@ from openhcs.core.artifacts.specs import ArtifactSpec from openhcs.core.artifacts.kinds import ArtifactType from openhcs.core.component_group_scope import ( - ComponentGroupScope, + GroupedScope, RuntimeExecutionAxisScope, ) @@ -72,11 +72,7 @@ def from_output_plan( raise TypeError( "RuntimeValue execution_scope must be a RuntimeExecutionAxisScope." ) - group_key = output_plan.single_group_key - artifact_scope = execution_scope.for_group_coordinate( - output_plan.group_component if group_key is not None else None, - group_key, - ) + artifact_scope = execution_scope.for_output_scope(output_plan.group_scope) return RuntimeValue.from_spec( output_plan, data, @@ -105,44 +101,43 @@ def from_spec( ) @classmethod - def compose( + def compose(cls, values: Sequence["RuntimeValue"]) -> Any: + """Compose records through their shared artifact-type owner.""" + + runtime_values = cls._one_artifact_type(values) + return runtime_values[0].artifact_type.compose_runtime_values(runtime_values) + + @classmethod + def compose_groups( cls, values: Sequence["RuntimeValue"], - producer_group_scope: ComponentGroupScope | None = None, + producer_scope: GroupedScope, ) -> Any: - """Compose producer records through their shared artifact-type owner.""" + """Rebuild the producer's group axis from one record per producer group.""" + + runtime_values = cls._one_artifact_type(values) + for value in runtime_values: + value_scope = value.key.scope + if value_scope.component is not producer_scope.component: + raise ValueError( + "Runtime producer-group composition received a value from " + f"{value_scope.component!r}, expected " + f"{producer_scope.component!r}." + ) + if not producer_scope.contains_runtime_key(value_scope.value_text): + raise ValueError( + "Runtime producer-group composition received a value outside " + f"the declared scope {producer_scope!r}." + ) + return runtime_values[0].artifact_type.compose_runtime_groups(runtime_values) + @staticmethod + def _one_artifact_type( + values: Sequence["RuntimeValue"], + ) -> tuple["RuntimeValue", ...]: runtime_values = tuple(values) if not runtime_values: raise ValueError("Cannot compose an empty runtime value group.") - if producer_group_scope is not None and not isinstance( - producer_group_scope, ComponentGroupScope - ): - raise TypeError( - "RuntimeValue.compose producer_group_scope must be a " - "ComponentGroupScope or None." - ) - if producer_group_scope is not None: - if producer_group_scope.is_ungrouped: - raise ValueError( - "Runtime producer-group composition requires a grouped producer " - "scope." - ) - for value in runtime_values: - value_scope = value.key.scope - if value_scope.component is not producer_group_scope.component: - raise ValueError( - "Runtime producer-group composition received a value from " - f"{value_scope.component!r}, expected " - f"{producer_group_scope.component!r}." - ) - if not producer_group_scope.contains_runtime_key( - value_scope.value_text - ): - raise ValueError( - "Runtime producer-group composition received a value outside " - f"the declared scope {producer_group_scope!r}." - ) artifact_type = runtime_values[0].artifact_type mismatched_types = tuple( value.artifact_type @@ -154,10 +149,7 @@ def compose( "Grouped runtime values must have one artifact type, got " f"{(artifact_type, *mismatched_types)!r}." ) - return artifact_type.compose_runtime_values( - runtime_values, - producer_group_scope=producer_group_scope, - ) + return runtime_values @staticmethod def normalize( @@ -214,14 +206,7 @@ def normalize_for_execution_scope( raise TypeError( "RuntimeValue execution_scope must be a RuntimeExecutionAxisScope." ) - expected_scope = execution_scope.for_group_coordinate( - ( - output_plan.group_component - if output_plan.single_group_key is not None - else None - ), - output_plan.single_group_key, - ) + expected_scope = execution_scope.for_output_scope(output_plan.group_scope) if isinstance(value, RuntimeValue): validated = value.validated_for_output_plan( output_plan, diff --git a/openhcs/core/runtime_stores.py b/openhcs/core/runtime_stores.py index edcf387eec..fa92e05de1 100644 --- a/openhcs/core/runtime_stores.py +++ b/openhcs/core/runtime_stores.py @@ -41,7 +41,6 @@ SourceAxisMetadataScope, SourceImageSetIdentityCompatibility, SourceImageSetIdentityPolicy, - semantic_source_metadata_value, ) from openhcs.core.source_bindings import CompiledSourceBindingPlan from openhcs.core.axes import Axis, PartitionAxis @@ -136,7 +135,7 @@ def __post_init__(self) -> None: def matches(self, record: "StoredRuntimeValue") -> bool: scope = record.key.scope - producer_scope = self.input_plan.producer_group_scope() + producer_scope = self.input_plan.group_scope if scope.component is not producer_scope.component or scope.value_text is None: return False return record.location == RuntimeArtifactLocation( @@ -205,11 +204,7 @@ def from_input_plan( group_key: str | None = None, ) -> "RuntimeArtifactQuery": """Build the runtime-store query for one compiled artifact input.""" - input_scope = ComponentGroupScope.from_raw( - input_plan.group_keys, - component=input_plan.group_component, - ) - if input_scope.is_dynamic and group_key is None: + if group_key is None and not input_plan.group_scope.has_compiled_keys: return cls( name=input_plan.name, artifact_type=input_plan.artifact_type, @@ -321,9 +316,7 @@ def records( storage_plan = self.edge_plan.storage_plan projection = self.edge_plan.projection - producer_scope = storage_plan.producer_group_scope() - if producer_scope.is_ungrouped: - return self._records(store, producer_scope, None) + producer_scope = storage_plan.group_scope selection_scope = projection.producer_selection_scope if self.selects_complete_producer(): return self.all_records(store) @@ -341,7 +334,7 @@ def selects_complete_producer(self) -> bool: storage_plan = self.edge_plan.storage_plan projection = self.edge_plan.projection - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope selection_scope = projection.producer_selection_scope return ( projection.selects_declared_complete_producer(self.edge_plan.spec, storage_plan) @@ -350,8 +343,7 @@ def selects_complete_producer(self) -> bool: ) or ( selection_scope == producer_scope - and not selection_scope.is_dynamic - and len(selection_scope.keys) > 1 + and selection_scope.names_several_keys ) ) @@ -369,7 +361,7 @@ def candidate_execution_scopes( """ storage_plan = self.edge_plan.storage_plan - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope projection = self.edge_plan.projection selection_scope = projection.producer_selection_scope selects_complete_producer = self.selects_complete_producer() @@ -377,7 +369,7 @@ def candidate_execution_scopes( identity_policy = self._source_context_identity_policy() candidates = tuple( record - for group_key in ((None,) if producer_scope.is_dynamic else producer_scope.keys) + for group_key in producer_scope.keys for record in store.find_matching( RuntimeArtifactQuery.from_input_plan( storage_plan, @@ -413,16 +405,11 @@ def candidate_execution_scopes( continue coordinates = dict(record_coordinates) for component in projected_components: - component_scope = projection.component_scope(component) - if component_scope.is_dynamic: - component_value = semantic_source_metadata_value( - metadata, component.name, - ) - if component_value is None: - break - component_value = str(component_value) - else: - component_value = component_scope.select_runtime_key(None) + component_value = projection.component_scope( + component + ).value_for_plane(metadata, component) + if component_value is None: + break if not SourceAxisMetadataScope.constraint_matches_metadata( metadata, component.name, component_value, ): @@ -435,15 +422,14 @@ def candidate_execution_scopes( coordinates.setdefault(component, component_value) else: runtime_value = coordinates.get(execution_scope.component) - if execution_scope.is_dynamic and runtime_value is None: - continue - component_keys = execution_scope.runtime_keys((runtime_value,)) - for component_key in component_keys: + for component_key in execution_scope.candidate_keys(runtime_value): if ( not selects_complete_producer - and selection_scope.is_dynamic - and selection_scope.component is execution_scope.component - and record_scope.value_text != component_key + and not selection_scope.admits_record_key( + execution_scope.component, + component_key, + record_scope.value_text, + ) ): continue fixed_group_value = coordinates.get(execution_scope.component) @@ -473,8 +459,8 @@ def all_records( """Resolve every producer group explicitly declared by this plan.""" storage_plan = self.edge_plan.storage_plan - producer_scope = storage_plan.producer_group_scope() - if not producer_scope.is_dynamic: + producer_scope = storage_plan.group_scope + if producer_scope.has_compiled_keys: return tuple( record for group_key in producer_scope.keys @@ -523,10 +509,7 @@ def composed_value( ) -> Any: """Compose exact records after projecting the runtime-axis coordinate.""" - return RuntimeValue.compose( - tuple(self._axis_value(record) for record in records), - self._producer_group_composition_scope(), - ) + return self._compose(tuple(self._axis_value(record) for record in records)) def projected_values( self, @@ -539,23 +522,21 @@ def projected_values( def resolve_value(self, store: "RuntimeValueStore") -> Any: """Resolve this compiled input to its invocation value.""" - return RuntimeValue.compose( - self.projected_values(store), - self._producer_group_composition_scope(), - ) + return self._compose(self.projected_values(store)) - def _producer_group_composition_scope( - self, - ) -> ComponentGroupScope | None: - """Return declared group-axis reconstruction semantics for this input.""" + def _compose(self, values: tuple[RuntimeValue, ...]) -> Any: + """Compose plainly, or rebuild the producer's group axis when consumed.""" storage_plan = self.edge_plan.storage_plan consumer_components = ComponentSet.of( self.edge_plan.projection.consumer_variable_components ) if not storage_plan.composes_producer_groups(consumer_components): - return None - return storage_plan.producer_group_scope() + return RuntimeValue.compose(values) + return RuntimeValue.compose_groups( + values, + storage_plan.group_scope.require_grouped("Producer-group composition"), + ) def _axis_value(self, value: RuntimeValue) -> RuntimeValue: projected_components = ComponentSet.of( @@ -727,7 +708,7 @@ def _matches_execution_scope( ).excluding( ComponentSet.collect( projected_components, - (self.edge_plan.storage_plan.producer_group_scope().component,), + (self.edge_plan.storage_plan.group_scope.component,), ) ) identity_policy = self._source_context_identity_policy() @@ -979,14 +960,15 @@ def address_matches_plan( return False if key.scope.axis_id != axis_id: return False - if key.scope.component is not plan.group_component: + if key.scope.component is not plan.group_scope.component: + return False + resolved_plan = ( + plan.for_group(key.scope.value_text) + if plan.group_scope.as_grouped() + else plan + ) + if resolved_plan is None: return False - if plan.group_component is None: - resolved_plan = plan - else: - resolved_plan = plan.for_group(key.scope.value_text) - if resolved_plan is None: - return False return address.location.path == resolved_plan.path @property diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index ae912d7b9a..3aaf5669d0 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -379,15 +379,9 @@ def actual_materialization_records( output_plan: ArtifactOutputPlan, ) -> tuple[StoredRuntimeValue, ...]: """Return actually produced memory records for one planned artifact output.""" - if not output_plan.group_keys: - raise RuntimeError( - f"Artifact output plan '{output_plan.name}' has no group keys." - ) artifact_type = output_plan.artifact_type - if output_plan.group_component is not None and tuple(output_plan.group_keys) == ( - None, - ): + if not output_plan.group_scope.has_compiled_keys: dynamic_records = tuple( record for record in store.find( @@ -434,11 +428,12 @@ def actual_materialization_records( ) group_order = { - group_key: index for index, group_key in enumerate(output_plan.group_keys) + group_key: index + for index, group_key in enumerate(output_plan.group_scope.keys) } record_sort_items = [] missing_group_keys = [] - for group_key in output_plan.group_keys: + for group_key in output_plan.group_scope.keys: records = tuple( record for record in store.find( @@ -1161,7 +1156,7 @@ def planned_materialization_preview( output_plan=output_plan.for_group(group_key), materialization_spec=materialization_spec, ) - for group_key in (output_plan.group_keys or (None,)) + for group_key in output_plan.group_scope.keys ) return PlannedArtifactMaterializationPreview( filename_uses_source_identity=( @@ -1184,14 +1179,14 @@ def _planned_materialization_path( output_key, plan, context, - output_plan.single_group_key, + output_plan.group_scope.single_key(), artifact_path=output_plan.path, output_plan=output_plan, ) base_path = str(plan.artifact_analysis_output_dir / filename) shared_output_stem = materialization_spec.shared_output_stem(base_path) return PlannedArtifactMaterializationPath( - group_key=output_plan.single_group_key, + group_key=output_plan.group_scope.single_key(), shared_output_stem=shared_output_stem, candidate_paths=materialization_spec.candidate_paths(base_path), ) diff --git a/openhcs/core/steps/function_execution.py b/openhcs/core/steps/function_execution.py index feb3070afc..45c6668675 100644 --- a/openhcs/core/steps/function_execution.py +++ b/openhcs/core/steps/function_execution.py @@ -593,7 +593,7 @@ def candidate_scopes( variable_components=ComponentSet.of(plan.variable_components), ) - if scope.is_dynamic: + if not scope.has_compiled_keys: discovered_keys = [] for group in pattern.groups: for invocation in group.invocations: @@ -634,9 +634,7 @@ def candidate_scopes( for edge in edges: edge_cohorts = candidate_scopes( edge, - ComponentGroupScope.from_raw( - (component_key,), component=scope.component - ), + ComponentGroupScope.of((component_key,), component=scope.component), ) if not edge_cohorts: raise ValueError( @@ -972,7 +970,7 @@ def _prepare_groups( "Ungrouped is only valid for non-dict function patterns." ) if ( - plan.execution_group_scope.is_ungrouped + not plan.execution_group_scope.as_grouped() and not plan.compiled_function_pattern.is_grouped ): axis_patterns = _single_execution_group_patterns(axis_patterns) diff --git a/openhcs/interop/cellprofiler/runtime/output_record_request.py b/openhcs/interop/cellprofiler/runtime/output_record_request.py index 6db5f65290..24ceec7f9b 100644 --- a/openhcs/interop/cellprofiler/runtime/output_record_request.py +++ b/openhcs/interop/cellprofiler/runtime/output_record_request.py @@ -182,10 +182,10 @@ def measurement_source_metadata( raise ValueError("Measurement source context requires declared artifacts.") artifact_values = tuple(self.declared_artifact_value(spec) for spec in specs) metadata = tuple(value.metadata for value in artifact_values) - source_group_component = self.output_plan.group_component identity_policy = SourceImageSetIdentityPolicy( frozenset( - () if source_group_component is None else (source_group_component,) + grouped.component + for grouped in self.output_plan.group_scope.as_grouped() ) ) image_set_axes = tuple( diff --git a/openhcs/pyqt_gui/widgets/artifact_plan_view.py b/openhcs/pyqt_gui/widgets/artifact_plan_view.py index c080c4ac83..198b2d64b6 100644 --- a/openhcs/pyqt_gui/widgets/artifact_plan_view.py +++ b/openhcs/pyqt_gui/widgets/artifact_plan_view.py @@ -70,7 +70,7 @@ def required_text(self) -> str: def group_scope_text(self) -> str: if self.plan is None: return "" - scope = self.plan.group_scope() + scope = self.plan.group_scope component = "" if scope.component is None else scope.component.name keys = ", ".join("default" if key is None else key for key in scope.keys) return keys if not component else f"{component}: {keys}" diff --git a/tests/integration/test_chained_callable_lineage_journey.py b/tests/integration/test_chained_callable_lineage_journey.py index 9abae056aa..96d3788bdd 100644 --- a/tests/integration/test_chained_callable_lineage_journey.py +++ b/tests/integration/test_chained_callable_lineage_journey.py @@ -156,7 +156,7 @@ def test_chained_public_callable_uses_declared_main_flow_not_storage_argument( [record] = context.runtime_value_store.find_matching( RuntimeArtifactQuery.from_output_plan( plan, axis_id="A01", backend="memory", - group_key=plan.group_scope().select_runtime_key( + group_key=plan.group_scope.select_runtime_key( invocation.key.group_key ), ) diff --git a/tests/unit/agent/test_agent_services.py b/tests/unit/agent/test_agent_services.py index 16eb1a4b7b..da07ba1311 100644 --- a/tests/unit/agent/test_agent_services.py +++ b/tests/unit/agent/test_agent_services.py @@ -326,7 +326,7 @@ def _compile(self, request): output_dir=Path("/tmp/out/A01"), main_flow_axis_persistence_enabled=False, ) - step_plan.execution_group_scope = ComponentGroupScope.from_raw( + step_plan.execution_group_scope = ComponentGroupScope.of( ("A01",), component=Microscopy.Well, ) @@ -346,8 +346,7 @@ def _compile(self, request): name="positions", artifact_type=SpecialArtifactType, path="/tmp/out/A01/positions.pkl", - group_component=Microscopy.Well, - group_keys=("A01",), + group_scope=ComponentGroupScope.of(("A01",), component=Microscopy.Well), paths_by_group={"A01": "/tmp/out/A01/positions.pkl"}, source_step_id=0, source_step_scope_id="step-find-positions", @@ -357,8 +356,7 @@ def _compile(self, request): name="objects", artifact_type=ObjectLabelsArtifactType, path="/tmp/out/A01/objects.zarr", - group_component=Microscopy.Well, - group_keys=("A01",), + group_scope=ComponentGroupScope.of(("A01",), component=Microscopy.Well), paths_by_group={"A01": "/tmp/out/A01/objects.zarr"}, ) step_plan.artifact_outputs[output_plan.ref()] = output_plan diff --git a/tests/unit/progress/test_live_measurements.py b/tests/unit/progress/test_live_measurements.py index 144cca4bae..a11400d112 100644 --- a/tests/unit/progress/test_live_measurements.py +++ b/tests/unit/progress/test_live_measurements.py @@ -32,6 +32,9 @@ StepCompletedProgressPhase, SuccessProgressStatus, ) +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) @dataclass(frozen=True, slots=True) @@ -109,8 +112,7 @@ def _measurement_record( name=name, path=path, artifact_type=MeasurementsArtifactType, - group_keys=("DAPI",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Channel), ), MeasurementTable( name=name, diff --git a/tests/unit/test_aligned_main_flow_projection.py b/tests/unit/test_aligned_main_flow_projection.py index 7f17d926a8..8bbf208607 100644 --- a/tests/unit/test_aligned_main_flow_projection.py +++ b/tests/unit/test_aligned_main_flow_projection.py @@ -33,6 +33,9 @@ from openhcs.processing.backends.cellprofiler.color import color_to_gray from openhcs.domains.microscopy.axes import Microscopy from openhcs.core.runtime_image_values import PlainImagePayload +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) class _CompiledInputRequest: @@ -175,8 +178,7 @@ def _compiled_image_output( path=f"/memory/{spec.name}.pkl", artifact_type=spec.artifact_type, relations=spec.relations, - group_keys=(channel,), - group_component=(None if channel is None else Microscopy.Channel), + group_scope=ComponentGroupScope.of((channel,), component=None if channel is None else Microscopy.Channel), ) diff --git a/tests/unit/test_artifact_input_edge_cardinality.py b/tests/unit/test_artifact_input_edge_cardinality.py index 07ba54258e..52c033c00e 100644 --- a/tests/unit/test_artifact_input_edge_cardinality.py +++ b/tests/unit/test_artifact_input_edge_cardinality.py @@ -7,7 +7,10 @@ from openhcs.core.artifacts.specs import ArtifactSpec from openhcs.core.artifacts.relations import InputGroupLineageSourceRelation from openhcs.core.artifacts.kinds import ObjectLabelsArtifactType -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.component_set import ComponentSet from openhcs.core.callable_contract import CallableContract from openhcs.core.function_patterns import compile_function_pattern @@ -77,9 +80,9 @@ def consume_repeated_objects(image): compiled, artifact_inputs={storage_plan.ref(): storage_plan}, relation_source_scopes={ - objects.ref(): storage_plan.producer_group_scope(), + objects.ref(): storage_plan.group_scope, }, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), ) invocation = next(compiled.iter_invocations()) @@ -118,8 +121,7 @@ def consume_ambiguous_objects(image): name=source.name, path="/memory/source.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), ) compiled = compile_function_pattern( admitted_graph( @@ -142,10 +144,10 @@ def consume_ambiguous_objects(image): source_plan.ref(): source_plan, }, relation_source_scopes={ - plain_objects.ref(): object_plan.producer_group_scope(), - source.ref(): source_plan.producer_group_scope(), + plain_objects.ref(): object_plan.group_scope, + source.ref(): source_plan.group_scope, }, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), ) invocation = next(compiled.iter_invocations()) @@ -162,9 +164,9 @@ def consume_ambiguous_objects(image): ) assert invocation.artifact_input_edges[0].projection.component_scopes == () assert invocation.artifact_input_edges[1].projection.component_scopes == ( - source_plan.producer_group_scope(), + source_plan.group_scope, ) assert ( invocation.artifact_input_edges[2].projection.producer_selection_scope - == source_plan.producer_group_scope() + == source_plan.group_scope ) diff --git a/tests/unit/test_artifact_input_projection_plan.py b/tests/unit/test_artifact_input_projection_plan.py index ae982a0382..796c02186e 100644 --- a/tests/unit/test_artifact_input_projection_plan.py +++ b/tests/unit/test_artifact_input_projection_plan.py @@ -7,7 +7,11 @@ MeasurementsArtifactType, ObjectLabelsArtifactType, ) -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + DynamicGroupScope, + UngroupedScope, +) from openhcs.core.component_set import ComponentSet from openhcs.core.axes import Axis from openhcs.domains.microscopy.axes import Microscopy @@ -32,8 +36,8 @@ def _projection( @pytest.mark.parametrize( "invocation_scope", ( - ComponentGroupScope.ungrouped(), - ComponentGroupScope.dynamic(Microscopy.Channel), + UngroupedScope(), + DynamicGroupScope(Microscopy.Channel), ), ids=("ungrouped-invocation", "dynamic-dispatch-invocation"), ) @@ -44,13 +48,12 @@ def test_grouped_producer_selection_domain_is_independent_of_invocation_dispatch name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), ) projection = _projection( storage_plan, invocation_scope=invocation_scope, - producer_selection_scope=storage_plan.producer_group_scope(), + producer_selection_scope=storage_plan.group_scope, consumer_variable_components=(Microscopy.Site,), ) @@ -62,13 +65,12 @@ def test_dynamic_producer_coordinate_requires_matching_invocation_dispatch() -> name="objects", path="/memory/objects.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), ) projection = _projection( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Site), - producer_selection_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + invocation_scope=DynamicGroupScope(Microscopy.Site), + producer_selection_scope=DynamicGroupScope(Microscopy.Channel), consumer_variable_components=(Microscopy.Timepoint,), ) @@ -83,15 +85,14 @@ def test_existing_producer_stack_is_retained_when_consumer_relabels_third_axis() name="objects", path="/memory/objects.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ) consumer_components = ComponentSet((Microscopy.Timepoint,)) projection = _projection( storage_plan, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=storage_plan.producer_group_scope(), + invocation_scope=UngroupedScope(), + producer_selection_scope=storage_plan.group_scope, consumer_variable_components=consumer_components.as_tuple(), ) @@ -111,14 +112,13 @@ def test_scalar_consumer_requires_coordinate_for_each_producer_stack_component() name="objects", path="/memory/objects.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ) projection = _projection( storage_plan, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=storage_plan.producer_group_scope(), + invocation_scope=UngroupedScope(), + producer_selection_scope=storage_plan.group_scope, ) with pytest.raises(ValueError, match="site.*without an exact coordinate"): @@ -130,16 +130,15 @@ def test_transposed_producer_group_axis_requires_old_stack_coordinate() -> None: name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ) consumer_components = ComponentSet((Microscopy.Channel,)) projection = _projection( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Channel), - producer_selection_scope=storage_plan.producer_group_scope(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Channel),), + invocation_scope=DynamicGroupScope(Microscopy.Channel), + producer_selection_scope=storage_plan.group_scope, + component_scopes=(DynamicGroupScope(Microscopy.Channel),), consumer_variable_components=consumer_components.as_tuple(), ) @@ -158,16 +157,15 @@ def test_transposed_producer_stack_rejects_multi_coordinate_projection() -> None name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ) projection = _projection( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Channel), - producer_selection_scope=storage_plan.producer_group_scope(), + invocation_scope=DynamicGroupScope(Microscopy.Channel), + producer_selection_scope=storage_plan.group_scope, component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2", "3"), component=Microscopy.Site, ), diff --git a/tests/unit/test_artifact_plan_view.py b/tests/unit/test_artifact_plan_view.py index 6501a90a79..8747139209 100644 --- a/tests/unit/test_artifact_plan_view.py +++ b/tests/unit/test_artifact_plan_view.py @@ -28,6 +28,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.debug import DebugArtifactRef, DebugCursor, DebugSnapshot @@ -112,8 +113,8 @@ def transform(image, labels): spec=input_spec, storage_plan=input_plan, projection=ArtifactInputProjectionPlan( - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), ), ) invocation = CompiledFunctionInvocation( diff --git a/tests/unit/test_cellprofiler_area_occupied.py b/tests/unit/test_cellprofiler_area_occupied.py index 00d1211b59..a5ae6f5892 100644 --- a/tests/unit/test_cellprofiler_area_occupied.py +++ b/tests/unit/test_cellprofiler_area_occupied.py @@ -15,6 +15,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.function_patterns import ( FunctionInvocationKey, @@ -60,8 +61,8 @@ def _binding_request( ), spec=spec, input_index=input_index, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) diff --git a/tests/unit/test_cellprofiler_artifact_declarations.py b/tests/unit/test_cellprofiler_artifact_declarations.py index e8fcee08c7..645f877dc1 100644 --- a/tests/unit/test_cellprofiler_artifact_declarations.py +++ b/tests/unit/test_cellprofiler_artifact_declarations.py @@ -626,12 +626,11 @@ def test_calculate_math_retains_measurement_subjects_across_producer_channels() name=measurements.name, path="/memory/intensity.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("2", "1"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2", "1"), component=Microscopy.Channel), ) assert ArtifactInputProjectionPlan.declared_producer_selection_scope( measurement_input, storage - ) == ComponentGroupScope.from_raw(("2", "1"), component=Microscopy.Channel) + ) == ComponentGroupScope.of(("2", "1"), component=Microscopy.Channel) def test_declarations_carry_cross_step_object_and_measurement_flow() -> None: diff --git a/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py b/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py index 8a45f03f3c..5230ea5de3 100644 --- a/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py +++ b/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py @@ -15,7 +15,10 @@ from openhcs.core.artifacts.specs import ArtifactSpec, ArtifactSpecCollection from openhcs.core.artifacts.kinds import ImageArtifactType, ObjectLabelsArtifactType from openhcs.core.callable_contract import CallableContract -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.component_set import ComponentSet from openhcs.core.function_patterns import ( compile_function_pattern, @@ -263,7 +266,7 @@ def test_authored_identity_remains_visible_through_compiled_artifact_graph() -> compiled, artifact_inputs={}, relation_source_scopes={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), source_bindings=source_bindings, available_artifacts=ArtifactSpecCollection(contract.artifact_inputs), diff --git a/tests/unit/test_cellprofiler_combineobjects_source_scope.py b/tests/unit/test_cellprofiler_combineobjects_source_scope.py index d7332dabe3..3eb9fbb960 100644 --- a/tests/unit/test_cellprofiler_combineobjects_source_scope.py +++ b/tests/unit/test_cellprofiler_combineobjects_source_scope.py @@ -24,7 +24,7 @@ from openhcs.core.pipeline.function_contracts import artifact_inputs, artifact_outputs from openhcs.core.pipeline.path_planner import ( PathPlannerExecutionGroups, - PathPlannerGroupScope, + ComponentGroupScope, ) from openhcs.core.progress import set_progress_queue from openhcs.core.source_bindings import NamedSourceBinding, SourceProjectionRole @@ -33,6 +33,9 @@ CellProfilerPlateWorkspacePreparer, ) from openhcs.domains.microscopy.axes import Microscopy +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) COMBINE_OBJECTS_CPIPE = """CellProfiler Pipeline: http://www.cellprofiler.org Version:5 @@ -109,13 +112,13 @@ def combine(object_labels): scope = PathPlannerExecutionGroups(planner).artifact_owned_execution_scope( snapshot, (CallableContract.from_callable(combine),), - consumer_scope=PathPlannerGroupScope.from_raw( + consumer_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -158,7 +161,7 @@ def test_public_combineobjects_import_compiles_source_owned_scope( step_plan = context.step_plans[0] assert tuple(step_plan.variable_components) == (Microscopy.Site,) assert step_plan.group_by is Microscopy.Channel - assert step_plan.execution_group_scope == PathPlannerGroupScope.from_raw( + assert step_plan.execution_group_scope == ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) diff --git a/tests/unit/test_cellprofiler_generic_special_input_binding.py b/tests/unit/test_cellprofiler_generic_special_input_binding.py index effdbc4508..9c30764825 100644 --- a/tests/unit/test_cellprofiler_generic_special_input_binding.py +++ b/tests/unit/test_cellprofiler_generic_special_input_binding.py @@ -13,7 +13,10 @@ from openhcs.core.artifacts.specs import ArtifactSpec, ArtifactSpecCollection from openhcs.core.artifacts.kinds import ImageArtifactType, ObjectLabelsArtifactType from openhcs.core.artifacts.relations import InputStackBroadcastSourceRelation -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.component_set import ComponentSet from openhcs.core.function_patterns import ( compile_function_pattern, @@ -180,7 +183,7 @@ def _compile_public_step(step: FunctionStep): compiled, artifact_inputs={}, relation_source_scopes={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), source_bindings=step.source_bindings, available_artifacts=ArtifactSpecCollection( @@ -209,9 +212,9 @@ def _compile_runtime_input_edges(func, plans: tuple[ArtifactInputPlan, ...]): compiled, artifact_inputs=plans_by_ref, relation_source_scopes={ - plan.ref(): plan.producer_group_scope() for plan in plans + plan.ref(): plan.group_scope for plan in plans }, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), ) diff --git a/tests/unit/test_cellprofiler_library_untangleworms.py b/tests/unit/test_cellprofiler_library_untangleworms.py index 0784ebcd61..5de63cea10 100644 --- a/tests/unit/test_cellprofiler_library_untangleworms.py +++ b/tests/unit/test_cellprofiler_library_untangleworms.py @@ -17,6 +17,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.processing.backends.cellprofiler.worms import ( OverlapStyle, @@ -108,8 +109,8 @@ def _straighten_worms_binding_request( artifact_type=object_spec.artifact_type, ), spec=object_spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) diff --git a/tests/unit/test_cellprofiler_measurement_slice_consumers.py b/tests/unit/test_cellprofiler_measurement_slice_consumers.py index 23879a46ff..74595a04bc 100644 --- a/tests/unit/test_cellprofiler_measurement_slice_consumers.py +++ b/tests/unit/test_cellprofiler_measurement_slice_consumers.py @@ -11,6 +11,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.measurement_feature_queries import MeasurementFeatureQuery from openhcs.core.measurement_row_materialization import ( @@ -62,8 +63,8 @@ def __init__( edge = cellprofiler_runtime_input_edge_for_test( input_plan, spec=measurement_spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) diff --git a/tests/unit/test_cellprofiler_measurement_source_group_scope.py b/tests/unit/test_cellprofiler_measurement_source_group_scope.py index f2b46440e2..477a6687fd 100644 --- a/tests/unit/test_cellprofiler_measurement_source_group_scope.py +++ b/tests/unit/test_cellprofiler_measurement_source_group_scope.py @@ -14,7 +14,10 @@ ObjectLabelsArtifactType, ) from openhcs.core.callable_contract import CallableContract -from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + RuntimeExecutionAxisScope, +) from openhcs.core.function_patterns import ( DEFAULT_GROUP_KEY, FunctionInvocationKey, @@ -93,8 +96,7 @@ def combine_objects(object_labels): name=measurements.name, path="/artifacts/CombinedMeasurements", artifact_type=measurements.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), ) adapter = CellProfilerRuntimeAdapter( runtime_adapter_request_for_test( diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index 8cb157e124..ef8cc159d3 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -51,6 +51,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.config import DtypeConfig from openhcs.core.compiled_step_plan import CompiledStepPlan @@ -497,7 +498,7 @@ def test_default_invocation_keeps_compiled_source_bindings_outside_anchor_group( artifact_outputs=(measurement_spec,), ) executor = _module_executor(contract) - channel_scope = ComponentGroupScope( + channel_scope = ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) @@ -516,7 +517,7 @@ def test_default_invocation_keeps_compiled_source_bindings_outside_anchor_group( artifact_type=object_spec.artifact_type, ), invocation_scope=channel_scope, - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=(channel_scope,), consumer_variable_components=(), ), @@ -553,7 +554,7 @@ def test_active_inputs_restore_repeated_contract_roles_from_unique_storage() -> artifact_inputs=(objects, objects), ) executor = _module_executor(contract) - channel_scope = ComponentGroupScope(("1",), component=Microscopy.Channel) + channel_scope = ComponentGroupScope.of(("1",), component=Microscopy.Channel) runtime = _FakeCellProfilerRuntime( {}, artifact_input_edges=( @@ -564,7 +565,7 @@ def test_active_inputs_restore_repeated_contract_roles_from_unique_storage() -> artifact_type=objects.artifact_type, ), invocation_scope=channel_scope, - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=(channel_scope,), consumer_variable_components=(), ), @@ -605,7 +606,7 @@ def test_active_inputs_exclude_unselected_independent_contract_roles() -> None: artifact_inputs=(first, second), ) executor = _module_executor(contract) - channel_scope = ComponentGroupScope(("1",), component=Microscopy.Channel) + channel_scope = ComponentGroupScope.of(("1",), component=Microscopy.Channel) runtime = _FakeCellProfilerRuntime( {}, artifact_input_edges=( @@ -616,7 +617,7 @@ def test_active_inputs_exclude_unselected_independent_contract_roles() -> None: artifact_type=first.artifact_type, ), invocation_scope=channel_scope, - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=(channel_scope,), consumer_variable_components=(), ), @@ -753,7 +754,7 @@ def _artifact_input_edge_for_test( invocation_scope: ComponentGroupScope | None = None, stored: bool = True, ) -> InvocationArtifactInputEdgePlan: - scope = invocation_scope or ComponentGroupScope.ungrouped() + scope = invocation_scope or UngroupedScope() edge = replace( cellprofiler_runtime_input_edge_for_test( ArtifactInputPlan( @@ -764,7 +765,7 @@ def _artifact_input_edge_for_test( ), spec=spec, invocation_scope=scope, - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=() if scope.is_ungrouped else (scope,), consumer_variable_components=(), ), @@ -2563,8 +2564,8 @@ def test_track_objects_retained_image_uses_tracked_object_source_payload() -> No path="/artifacts/Embryos", artifact_type=ObjectLabelsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2731,8 +2732,8 @@ def test_object_label_output_source_preserves_matching_input_object_plane_contex path="/artifacts/Guides", artifact_type=ObjectLabelsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2796,8 +2797,8 @@ def test_resize_objects_output_source_policy_uses_input_object_context() -> None path="/artifacts/ReferenceImage", artifact_type=ImageArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2807,8 +2808,8 @@ def test_resize_objects_output_source_policy_uses_input_object_context() -> None path="/artifacts/Nuclei", artifact_type=ObjectLabelsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2932,8 +2933,8 @@ def record_native_value(name, kind, labels, **kwargs): path="/artifacts/ReferenceImage", artifact_type=ImageArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2943,8 +2944,8 @@ def record_native_value(name, kind, labels, **kwargs): path="/artifacts/Nuclei", artifact_type=ObjectLabelsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -3082,8 +3083,8 @@ def test_measure_object_size_shape_record_builder_keeps_shape_features_unqualifi path="/artifacts/Cells", artifact_type=ObjectLabelsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -3444,8 +3445,8 @@ def test_object_label_output_source_payload_uses_primary_object_input() -> None: path=f"/artifacts/{spec.name}", artifact_type=spec.artifact_type, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -3507,8 +3508,8 @@ def test_object_label_output_source_payload_uses_declared_primary_object_input() path="/artifacts/PrimaryObjects", artifact_type=ObjectLabelsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -4303,8 +4304,7 @@ def _storage_seed_plans( name=storage_plan.name, path=storage_plan.path_for_runtime_query(group_key), artifact_type=storage_plan.artifact_type, - group_keys=(group_key,), - group_component=storage_plan.group_component, + group_scope=ComponentGroupScope.of((group_key,), component=storage_plan.group_component), variable_components=storage_plan.variable_components, component_domains=storage_plan.component_domains, paths_by_group={ @@ -5066,8 +5066,8 @@ def test_adapter_object_record_uses_nominal_full_stack_binding() -> None: artifact_type=ObjectLabelsArtifactType, ), spec=object_spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -13538,7 +13538,7 @@ def test_object_measurement_table_uses_provenance_without_image_ownership() -> N output_plans=( replace( _artifact_output_plan(measurement_spec), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), ), ), spec=measurement_spec, @@ -13555,20 +13555,19 @@ def test_object_measurement_table_uses_provenance_without_image_ownership() -> N name=spec.name, path=f"/memory/{spec.name}.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), paths_by_group={ channel: f"/memory/{spec.name}_{channel}.pkl" }, ), spec=spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( (channel,), component=Microscopy.Channel, ), component_scopes=( - ComponentGroupScope( + ComponentGroupScope.of( (channel,), component=Microscopy.Channel, ), @@ -13756,17 +13755,16 @@ def test_relate_objects_measurement_table_rejects_distinct_image_set_axes() -> N name=name, path=f"/memory/{name}.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), paths_by_group={channel: f"/memory/{name}_{channel}.pkl"}, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( (channel,), component=Microscopy.Channel, ), component_scopes=( - ComponentGroupScope( + ComponentGroupScope.of( (channel,), component=Microscopy.Channel, ), diff --git a/tests/unit/test_cellprofiler_runtime_adapter.py b/tests/unit/test_cellprofiler_runtime_adapter.py index d9db25735a..3b2aa24012 100644 --- a/tests/unit/test_cellprofiler_runtime_adapter.py +++ b/tests/unit/test_cellprofiler_runtime_adapter.py @@ -31,7 +31,9 @@ ) from openhcs.core.component_group_scope import ( ComponentGroupScope, + DynamicGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.config import DtypeConfig, StepSourceBindingsConfig from openhcs.core.function_patterns import ( @@ -219,8 +221,8 @@ def _runtime_request_input_edge( artifact_type=ImageArtifactType, ), input_index=input_index, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -603,7 +605,7 @@ def _plan(name, kind, *, group_component=None): name=name, path=f"/memory/{name}.pkl", artifact_type=kind, - group_component=group_component, + group_scope=ComponentGroupScope.of((None,), component=group_component), ) @@ -883,8 +885,8 @@ def _executor_for_contract( ), spec=spec, input_index=input_index, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -1300,13 +1302,13 @@ def test_cellprofiler_adapter_does_not_cache_current_image_object_selection(): name=NUCLEI, path=outputs[NUCLEI].path, artifact_type=ObjectLabelsArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -1354,8 +1356,7 @@ def test_cellprofiler_adapter_does_not_select_relationship_from_current_source_p name=relationship_name, path="/memory/relationships.pkl", artifact_type=RelationshipsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=group_paths, ) filemanager = FileManagerStub() @@ -1396,16 +1397,15 @@ def test_cellprofiler_adapter_does_not_select_relationship_from_current_source_p name=relationship_name, path=output_plan.path, artifact_type=RelationshipsArtifactType, - group_keys=output_plan.group_keys, - group_component=output_plan.group_component, + group_scope=ComponentGroupScope.of(output_plan.group_keys, component=output_plan.group_component), paths_by_group=output_plan.paths_by_group, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -1458,8 +1458,7 @@ def test_cellprofiler_adapter_aligns_grouped_relationships_to_runtime_slices(): name=relationship_name, path="/memory/relationships.pkl", artifact_type=RelationshipsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=group_paths, ) filemanager = FileManagerStub() @@ -1496,16 +1495,15 @@ def test_cellprofiler_adapter_aligns_grouped_relationships_to_runtime_slices(): name=relationship_name, path=output_plan.path, artifact_type=RelationshipsArtifactType, - group_keys=output_plan.group_keys, - group_component=output_plan.group_component, + group_scope=ComponentGroupScope.of(output_plan.group_keys, component=output_plan.group_component), paths_by_group=output_plan.paths_by_group, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -1576,13 +1574,13 @@ def test_cellprofiler_adapter_does_not_source_scope_default_image_records(): name=DNA_IMAGE, path=outputs[DNA_IMAGE].path, artifact_type=ImageArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -1629,8 +1627,7 @@ def test_cellprofiler_adapter_stacks_declared_default_image_input_runtime_groups name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ) for group_key, source_path, value in ( @@ -1662,20 +1659,19 @@ def test_cellprofiler_adapter_stacks_declared_default_image_input_runtime_groups name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( (AXIS_ID,), component=Microscopy.Well ), - producer_selection_scope=ComponentGroupScope.dynamic( + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( (AXIS_ID,), component=Microscopy.Well ), - ComponentGroupScope.dynamic(Microscopy.Site), + DynamicGroupScope(Microscopy.Site), ), consumer_variable_components=(Microscopy.Site,), ), @@ -1732,8 +1728,7 @@ def test_cellprofiler_adapter_keeps_multisource_current_image_grouped_when_files name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ) for group_key, source_path, value in ( @@ -1764,14 +1759,13 @@ def test_cellprofiler_adapter_keeps_multisource_current_image_grouped_when_files name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -1822,8 +1816,7 @@ def test_cellprofiler_adapter_stacks_declared_image_input_for_pattern_group(): name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ) for group_key, source_path, value in ( @@ -1855,14 +1848,13 @@ def test_cellprofiler_adapter_stacks_declared_image_input_for_pattern_group(): name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -1910,8 +1902,7 @@ def test_cellprofiler_adapter_records_output_in_declared_invocation_group(): name=output_name, path=f"/memory/{output_name}.pkl", artifact_type=ImageArtifactType, - group_keys=("0", "3"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("0", "3"), component=Microscopy.Channel), paths_by_group={ "0": f"/memory/{output_name}_w0.pkl", "3": f"/memory/{output_name}_w3.pkl", @@ -1985,7 +1976,7 @@ def test_cellprofiler_adapter_projects_source_bound_runtime_image_to_group_plane name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), variable_components=(Microscopy.Site,), paths_by_group={None: image_path}, ), @@ -2019,16 +2010,16 @@ def test_cellprofiler_adapter_projects_source_bound_runtime_image_to_group_plane name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), paths_by_group={None: image_path}, variable_components=(Microscopy.Site,), ), - invocation_scope=ComponentGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Site ), - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=( - ComponentGroupScope.from_raw(("2",), component=Microscopy.Site), + ComponentGroupScope.of(("2",), component=Microscopy.Site), ), consumer_variable_components=(), ), @@ -2086,7 +2077,7 @@ def test_cellprofiler_adapter_deduplicates_grouped_runtime_image_input_locations name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), variable_components=(Microscopy.Site,), paths_by_group={None: image_path}, ), @@ -2120,16 +2111,16 @@ def test_cellprofiler_adapter_deduplicates_grouped_runtime_image_input_locations name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), paths_by_group={None: image_path}, variable_components=(Microscopy.Site,), ), - invocation_scope=ComponentGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Site ), - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=( - ComponentGroupScope.from_raw(("2",), component=Microscopy.Site), + ComponentGroupScope.of(("2",), component=Microscopy.Site), ), consumer_variable_components=(), ), @@ -2177,7 +2168,7 @@ def test_cellprofiler_adapter_uses_identity_record_for_collapsed_grouped_input() name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), paths_by_group={None: image_path}, ), ), @@ -2198,11 +2189,11 @@ def test_cellprofiler_adapter_uses_identity_record_for_collapsed_grouped_input() name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), paths_by_group={None: image_path}, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2238,8 +2229,7 @@ def test_cellprofiler_adapter_uses_grouped_input_when_consumer_group_is_differen name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group={ "1": "/memory/A01_w1_Mito.pkl", "2": "/memory/A01_w2_Mito.pkl", @@ -2268,24 +2258,23 @@ def test_cellprofiler_adapter_uses_grouped_input_when_consumer_group_is_differen name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group={ "1": "/memory/A01_w1_Mito.pkl", "2": "/memory/A01_w2_Mito.pkl", }, ), - invocation_scope=ComponentGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel ), - producer_selection_scope=ComponentGroupScope.from_raw( + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1",), component=Microscopy.Channel ), - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -2345,7 +2334,7 @@ def test_cellprofiler_adapter_does_not_project_channel_stack_for_site_group(): name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), variable_components=(Microscopy.Channel,), paths_by_group={None: image_path}, ), @@ -2379,17 +2368,17 @@ def test_cellprofiler_adapter_does_not_project_channel_stack_for_site_group(): name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), paths_by_group={None: image_path}, variable_components=(Microscopy.Channel,), ), - invocation_scope=ComponentGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Site ), - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=( - ComponentGroupScope.from_raw(("2",), component=Microscopy.Site), - ComponentGroupScope.from_raw( + ComponentGroupScope.of(("2",), component=Microscopy.Site), + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel ), ), @@ -2444,7 +2433,7 @@ def test_cellprofiler_adapter_projects_stack_without_replacing_artifact_provenan name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), variable_components=(Microscopy.Site,), paths_by_group={None: image_path}, ), @@ -2487,16 +2476,16 @@ def test_cellprofiler_adapter_projects_stack_without_replacing_artifact_provenan name=image_name, path=image_path, artifact_type=ImageArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), paths_by_group={None: image_path}, variable_components=(Microscopy.Site,), ), - invocation_scope=ComponentGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Site ), - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=( - ComponentGroupScope.from_raw(("2",), component=Microscopy.Site), + ComponentGroupScope.of(("2",), component=Microscopy.Site), ), consumer_variable_components=(), ), @@ -2547,8 +2536,7 @@ def test_cellprofiler_adapter_does_not_select_image_record_from_current_source_s name=image_name, path="/memory/Mito.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=image_paths, ) for group_key, site, value in ( @@ -2583,16 +2571,15 @@ def test_cellprofiler_adapter_does_not_select_image_record_from_current_source_s name=image_name, path="/memory/Mito.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=image_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -2697,13 +2684,13 @@ def test_cellprofiler_adapter_keeps_template_scoped_object_records_grouped(): name=NUCLEI, path=outputs[NUCLEI].path, artifact_type=ObjectLabelsArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -2750,8 +2737,8 @@ def test_cellprofiler_adapter_does_not_project_stacked_objects_from_source_metad path=output_plan.path, artifact_type=ObjectLabelsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2838,8 +2825,8 @@ def test_cellprofiler_adapter_preserves_nominal_object_label_spatial_domains(): path=source_plan.path, artifact_type=ImageArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2921,8 +2908,8 @@ def test_cellprofiler_adapter_rejects_non_nominal_object_label_values(labels): path=source_plan.path, artifact_type=ImageArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -2976,12 +2963,12 @@ def test_cellprofiler_adapter_reads_declared_inputs_by_compiled_location(): name=NUCLEI, path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), ) input_edge = cellprofiler_runtime_input_edge_for_test( input_plan, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -3022,8 +3009,7 @@ def test_cellprofiler_adapter_availability_accepts_grouped_runtime_inputs(): name=NUCLEI, path=path, artifact_type=ObjectLabelsArtifactType, - group_keys=(group_key,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Site), paths_by_group={group_key: path}, ), ), @@ -3046,16 +3032,15 @@ def test_cellprofiler_adapter_availability_accepts_grouped_runtime_inputs(): name=NUCLEI, path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=tuple(group_paths), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(tuple(group_paths), component=Microscopy.Site), paths_by_group=group_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -3101,8 +3086,7 @@ def test_cellprofiler_adapter_discovers_single_realized_dynamic_grouped_input(): name=NUCLEI, path=realized_path, artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": realized_path}, ), ), @@ -3125,15 +3109,14 @@ def test_cellprofiler_adapter_discovers_single_realized_dynamic_grouped_input(): name=NUCLEI, path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={"1": realized_path}, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Channel ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Channel),), + component_scopes=(DynamicGroupScope(Microscopy.Channel),), consumer_variable_components=(Microscopy.Channel,), ), ) @@ -3203,8 +3186,7 @@ def test_cellprofiler_adapter_discovers_realized_dynamic_grouped_object_inputs() name=CELLS, path=group_paths[group_key], artifact_type=ObjectLabelsArtifactType, - group_keys=(group_key,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Channel), paths_by_group={group_key: group_paths[group_key]}, ), ), @@ -3229,15 +3211,14 @@ def test_cellprofiler_adapter_discovers_realized_dynamic_grouped_object_inputs() name=CELLS, path="/memory/Cells.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group=group_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Channel ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Channel),), + component_scopes=(DynamicGroupScope(Microscopy.Channel),), consumer_variable_components=(Microscopy.Channel,), ), ) @@ -3290,8 +3271,7 @@ def test_cellprofiler_adapter_composes_object_input_across_declared_site_axis(): name=NUCLEI, path=group_paths[group_key], artifact_type=ObjectLabelsArtifactType, - group_keys=(group_key,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Site), paths_by_group={group_key: group_paths[group_key]}, ), ), @@ -3316,16 +3296,15 @@ def test_cellprofiler_adapter_composes_object_input_across_declared_site_axis(): name=NUCLEI, path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=group_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -3383,8 +3362,7 @@ def test_cellprofiler_adapter_does_not_resolve_object_input_from_source_context( name=NUCLEI, path=group_paths[group_key], artifact_type=ObjectLabelsArtifactType, - group_keys=(group_key,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Site), paths_by_group={group_key: group_paths[group_key]}, ), ), @@ -3410,16 +3388,15 @@ def test_cellprofiler_adapter_does_not_resolve_object_input_from_source_context( name=NUCLEI, path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=group_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -3502,10 +3479,10 @@ def test_cellprofiler_adapter_preserves_ungrouped_runtime_slice_output_stack(): artifact_type=ObjectLabelsArtifactType, variable_components=(Microscopy.Site,), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -3555,8 +3532,7 @@ def test_cellprofiler_adapter_stacks_dynamic_compiled_grouped_images(): name=DNA_IMAGE, path=group_paths["1"], artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), paths_by_group={"1": group_paths["1"]}, ), ), @@ -3575,8 +3551,7 @@ def test_cellprofiler_adapter_stacks_dynamic_compiled_grouped_images(): name=DNA_IMAGE, path=group_paths["2"], artifact_type=ImageArtifactType, - group_keys=("2",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Site), paths_by_group={"2": group_paths["2"]}, ), ), @@ -3593,15 +3568,14 @@ def test_cellprofiler_adapter_stacks_dynamic_compiled_grouped_images(): name=DNA_IMAGE, path="/memory/DNA.pkl", artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), paths_by_group=group_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -3672,11 +3646,11 @@ def test_cellprofiler_adapter_relationships_validate_declared_inputs_by_location name=NUCLEI, path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), ), input_index=0, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -3685,11 +3659,11 @@ def test_cellprofiler_adapter_relationships_validate_declared_inputs_by_location name=CELLS, path="/memory/Cells.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), ), input_index=1, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -3774,8 +3748,7 @@ def test_cellprofiler_adapter_relationships_accept_grouped_parent_inputs(): name=CELLS, path="/memory/Cells_s1.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), paths_by_group={"1": "/memory/Cells_s1.pkl"}, ), ), @@ -3794,8 +3767,7 @@ def test_cellprofiler_adapter_relationships_accept_grouped_parent_inputs(): name=CELLS, path="/memory/Cells_s2.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("2",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Site), paths_by_group={"2": "/memory/Cells_s2.pkl"}, ), ), @@ -3890,10 +3862,10 @@ def test_cellprofiler_adapter_relationships_allow_same_invocation_child_output() name=NUCLEI, path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), + group_scope=ComponentGroupScope.of((None,), component=None), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -4153,8 +4125,7 @@ def test_cellprofiler_adapter_uses_static_output_scope(): name=CELLS, path=output_path, artifact_type=ObjectLabelsArtifactType, - group_keys=("3",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Site), paths_by_group={"3": output_path}, ), ), @@ -4312,8 +4283,7 @@ def test_cellprofiler_adapter_does_not_select_measurement_record_from_current_so name=NUCLEI, path=object_group_paths[group_key], artifact_type=ObjectLabelsArtifactType, - group_keys=(group_key,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Site), paths_by_group={group_key: object_group_paths[group_key]}, ), ), @@ -4324,8 +4294,7 @@ def test_cellprofiler_adapter_does_not_select_measurement_record_from_current_so name=NUCLEI_MEASUREMENTS, path=group_paths[group_key], artifact_type=MeasurementsArtifactType, - group_keys=(group_key,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Site), paths_by_group={group_key: group_paths[group_key]}, ), ), @@ -4361,16 +4330,15 @@ def test_cellprofiler_adapter_does_not_select_measurement_record_from_current_so name=NUCLEI_MEASUREMENTS, path="/memory/NucleiMeasurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=group_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site ), ), @@ -4619,8 +4587,8 @@ def test_cellprofiler_adapter_hides_undeclared_same_name_object_table_occurrence path=output_plans[0].path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -4663,8 +4631,7 @@ def test_declared_measurement_inputs_require_producer_declared_slice_indexes(): name=MEASUREMENTS, path=f"/memory/{MEASUREMENTS}.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Site, - group_keys=tuple(group_paths), + group_scope=ComponentGroupScope.of(tuple(group_paths), component=Microscopy.Site), paths_by_group=group_paths, ) for index, value in enumerate((11.0, 13.0), start=1): @@ -4722,21 +4689,20 @@ def test_declared_measurement_inputs_require_producer_declared_slice_indexes(): name=MEASUREMENTS, path=output_plan.path, artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Site, - group_keys=output_plan.group_keys, + group_scope=ComponentGroupScope.of(output_plan.group_keys, component=Microscopy.Site), paths_by_group=output_plan.paths_by_group, ), - invocation_scope=ComponentGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel ), - producer_selection_scope=ComponentGroupScope.from_raw( + producer_selection_scope=ComponentGroupScope.of( output_plan.group_keys, component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1",), component=Microscopy.Channel ), - ComponentGroupScope.from_raw( + ComponentGroupScope.of( output_plan.group_keys, component=Microscopy.Site ), ), @@ -4836,13 +4802,13 @@ def test_cellprofiler_adapter_aligns_multiplane_measurements_across_groups(): name=NUCLEI_MEASUREMENTS, path=outputs[NUCLEI_MEASUREMENTS].path, artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -4935,8 +4901,8 @@ def test_cellprofiler_adapter_rejects_single_slice_measurements_for_repeated_lab path=measurement_output.path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -5118,8 +5084,8 @@ def test_source_qualified_object_feature_ignores_unrelated_runtime_slice_scope() path=measurement_output.path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -5264,13 +5230,13 @@ def test_cellprofiler_adapter_multiplane_measurement_alignment_is_feature_scoped name=NUCLEI_MEASUREMENTS, path=outputs[NUCLEI_MEASUREMENTS].path, artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.dynamic( + invocation_scope=UngroupedScope(), + producer_selection_scope=DynamicGroupScope( Microscopy.Site ), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Site,), ), ) @@ -5380,8 +5346,8 @@ def test_cellprofiler_adapter_measurement_query_cache_is_store_scoped(): path=measurement_output.path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -5559,8 +5525,8 @@ def test_label_measurement_cache_distinguishes_single_plane_from_stack() -> None path=single_measurement_output.path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -5589,8 +5555,8 @@ def test_label_measurement_cache_distinguishes_single_plane_from_stack() -> None path=measurement_output.path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -5657,8 +5623,8 @@ def test_cellprofiler_adapter_projects_duplicate_object_labels_to_current_runtim path=f"/memory/{NUCLEI_MEASUREMENTS}.pkl", artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -7033,8 +6999,7 @@ def test_relationship_plane_records_use_compiled_input_projection_order(): name=relationship_name, path=group_paths[group_key], artifact_type=RelationshipsArtifactType, - group_keys=(group_key,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Site), paths_by_group={group_key: group_paths[group_key]}, ), relationship, @@ -7049,16 +7014,15 @@ def test_relationship_plane_records_use_compiled_input_projection_order(): name=relationship_name, path="/memory/relationships.pkl", artifact_type=RelationshipsArtifactType, - group_keys=("site_a", "site_b"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("site_a", "site_b"), component=Microscopy.Site), paths_by_group=group_paths, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.from_raw( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("site_a", "site_b"), component=Microscopy.Site ), component_scopes=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("site_a", "site_b"), component=Microscopy.Site ), ), @@ -7342,8 +7306,8 @@ def test_adapter_measurement_vector_scope_uses_feature_bearing_axis_projection() path=measurement_output.path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -7453,8 +7417,8 @@ def test_adapter_multiplane_label_lookup_prefers_runtime_slice_axis(): path=measurement_output.path, artifact_type=MeasurementsArtifactType, ), - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), diff --git a/tests/unit/test_cellprofiler_runtime_cache_authorities.py b/tests/unit/test_cellprofiler_runtime_cache_authorities.py index 35835dab65..2fdd60e703 100644 --- a/tests/unit/test_cellprofiler_runtime_cache_authorities.py +++ b/tests/unit/test_cellprofiler_runtime_cache_authorities.py @@ -23,6 +23,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.callable_contract import CallableContract from openhcs.core.runtime_artifact_values import RuntimeValue @@ -86,8 +87,8 @@ def test_runtime_adapter_recomposes_images_from_runtime_value_store() -> None: cellprofiler_runtime_input_edge_for_test( input_plan, spec=input_spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -167,8 +168,8 @@ def test_runtime_adapter_measurement_queries_follow_store_revisions() -> None: cellprofiler_runtime_input_edge_for_test( input_plan, spec=input_spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ), @@ -284,8 +285,8 @@ def test_relationship_replacement_invalidates_cached_child_counts() -> None: relationship_edge = cellprofiler_runtime_input_edge_for_test( relationship_input_plan, spec=relationship_spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) diff --git a/tests/unit/test_cellprofiler_runtime_input_binding_contract.py b/tests/unit/test_cellprofiler_runtime_input_binding_contract.py index 569216924c..1abce826f6 100644 --- a/tests/unit/test_cellprofiler_runtime_input_binding_contract.py +++ b/tests/unit/test_cellprofiler_runtime_input_binding_contract.py @@ -24,6 +24,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.runtime_artifact_values import RuntimeValue from openhcs.core.runtime_image_values import ImagePayloadMetadata @@ -92,8 +93,8 @@ def consume(image: np.ndarray) -> np.ndarray: artifact_type=spec.artifact_type, ), spec=spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -129,8 +130,8 @@ def consume(image: np.ndarray) -> np.ndarray: artifact_type=second.artifact_type, ), spec=second, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -173,8 +174,8 @@ def consume(image: np.ndarray, mask: np.ndarray) -> np.ndarray: edge = cellprofiler_runtime_input_edge_for_test( input_plan, spec=spec, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -236,8 +237,8 @@ def consume(image: np.ndarray, mask: np.ndarray) -> np.ndarray: edge = cellprofiler_runtime_input_edge_for_test( input_plan, spec=mask, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(Microscopy.ZIndex,), ) @@ -399,8 +400,8 @@ def consume(image: np.ndarray, mask: np.ndarray) -> np.ndarray: edge = cellprofiler_runtime_input_edge_for_test( input_plan, spec=mask, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(Microscopy.ZIndex,), ) @@ -495,8 +496,8 @@ def consume( input_plan, spec=spec, input_index=input_index, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) @@ -592,8 +593,8 @@ def consume( input_plan, spec=spec, input_index=index - 1, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) diff --git a/tests/unit/test_cellprofiler_spreadsheet_export.py b/tests/unit/test_cellprofiler_spreadsheet_export.py index 0bd0fdfbe3..8a67108c1a 100644 --- a/tests/unit/test_cellprofiler_spreadsheet_export.py +++ b/tests/unit/test_cellprofiler_spreadsheet_export.py @@ -103,6 +103,9 @@ from openhcs.core.axes import Axis from openhcs.domains.microscopy.axes import Microscopy from tests.materialization_plans import output_plan as materialization_output_plan +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) def test_export_to_spreadsheet_declares_exact_plate_callable_abi() -> None: contract = CallableContract.from_callable(export_to_spreadsheet) @@ -1882,8 +1885,7 @@ def _measurement_record( name=name, path=f"/memory/{name}.pkl", artifact_type=MeasurementsArtifactType, - group_keys=(group_key,), - group_component=group_component, + group_scope=ComponentGroupScope.of((group_key,), component=group_component), variable_components=variable_components, ) value = RuntimeValue.normalize( diff --git a/tests/unit/test_child_measurement_producer_selection.py b/tests/unit/test_child_measurement_producer_selection.py index c4a91a6323..631bc68b78 100644 --- a/tests/unit/test_child_measurement_producer_selection.py +++ b/tests/unit/test_child_measurement_producer_selection.py @@ -24,7 +24,7 @@ ) from openhcs.core.pipeline.path_planner import ( PathPlannerArtifactStage, - PathPlannerGroupScope, + ComponentGroupScope, ) from openhcs.core.config import EMPTY_SOURCE_BINDINGS from openhcs.processing.backends.cellprofiler.relationships import RelateObjectsModule @@ -69,8 +69,7 @@ def _compiled_edge(*, dynamic=False, declared=True): name=spec.name, path="/memory/intensity.pkl", artifact_type=MeasurementsArtifactType, - group_keys=(None,) if dynamic else ("1", "2", "5", "3"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,) if dynamic else ("1", "2", "5", "3"), component=Microscopy.Channel), paths_by_group=( None if dynamic @@ -92,7 +91,7 @@ def _compiled_edge(*, dynamic=False, declared=True): InvocationArtifactInputProjectionKey(invocation.key, index), input_spec=spec, storage_plan=storage, - invocation_scope=PathPlannerGroupScope.from_raw( + invocation_scope=ComponentGroupScope.of( ("3",), component=Microscopy.Channel ), relation_source_scopes={}, @@ -108,14 +107,14 @@ def test_relate_prior_child_measurements_compile_all_source_channels(): edge, _ = _compiled_edge() assert ( edge.projection.producer_selection_scope - == edge.storage_plan.producer_group_scope() + == edge.storage_plan.group_scope ) assert edge.projection.consumer_variable_components == (Microscopy.Site,) def test_ordinary_measurement_input_still_selects_invocation_channel(): edge, _ = _compiled_edge(declared=False) - assert edge.projection.producer_selection_scope == ComponentGroupScope.from_raw( + assert edge.projection.producer_selection_scope == ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ) @@ -153,8 +152,7 @@ def _measurement_store(edge, *, well="A01", fixed=()): name="intensity", path=path, artifact_type=MeasurementsArtifactType, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), ) for subject in ("Children", "Unrelated"): table = MeasurementTable( @@ -374,8 +372,7 @@ def _upstream_rows(*, sites=("1", "2"), well="A01", time="1"): name=child.name, path="/memory/children.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), source_step_id=1, ) @@ -387,7 +384,7 @@ def _upstream_rows(*, sites=("1", "2"), well="A01", time="1"): storage_plan=storage, projection=replace( measurement_edge.projection, - producer_selection_scope=storage.producer_group_scope(), + producer_selection_scope=storage.group_scope, ), ) store = RuntimeValueStore() @@ -395,8 +392,7 @@ def _upstream_rows(*, sites=("1", "2"), well="A01", time="1"): name=child.name, path=storage.path, artifact_type=ObjectLabelsArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ) store.record( @@ -424,8 +420,7 @@ def _upstream_rows(*, sites=("1", "2"), well="A01", time="1"): name="intensity", path=path, artifact_type=MeasurementsArtifactType, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), ) for subject in ("Children", "Unrelated"): table = MeasurementTable( @@ -640,7 +635,7 @@ def test_plate_projection_keeps_complete_producer_scope(dynamic): invocation_key=edge.key.invocation_key, execution_scope=FunctionStepExecutionScope.PLATE, ) - assert scope == edge.storage_plan.producer_group_scope() + assert scope == edge.storage_plan.group_scope def test_compiled_input_cannot_silently_narrow_complete_declaration(): @@ -649,7 +644,7 @@ def test_compiled_input_cannot_silently_narrow_complete_declaration(): edge, projection=replace( edge.projection, - producer_selection_scope=ComponentGroupScope.from_raw( + producer_selection_scope=ComponentGroupScope.of( ("3",), component=Microscopy.Channel ), ), @@ -715,7 +710,7 @@ class SingleChannelRelation(ArtifactSpecRelation): target_plan_type = ArtifactInputPlan def input_producer_selection_scope(self, producer_scope): - return ComponentGroupScope.from_raw(("1",), component=Microscopy.Channel) + return ComponentGroupScope.of(("1",), component=Microscopy.Channel) spec = replace( edge.spec, relations=(*edge.spec.relations, SingleChannelRelation(child.ref())) diff --git a/tests/unit/test_classification_interop_boundaries.py b/tests/unit/test_classification_interop_boundaries.py index 9172ca28d5..5a5a9d78fd 100644 --- a/tests/unit/test_classification_interop_boundaries.py +++ b/tests/unit/test_classification_interop_boundaries.py @@ -25,6 +25,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.function_patterns import DEFAULT_GROUP_KEY, FunctionInvocationKey from openhcs.core.invocation_artifacts import ArtifactDeclarationStepContext @@ -116,8 +117,8 @@ def _runtime_request(contract, kwargs, table=None): plan, spec=spec, input_index=index, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), + producer_selection_scope=UngroupedScope(), component_scopes=(), consumer_variable_components=(), ) diff --git a/tests/unit/test_compiled_execution.py b/tests/unit/test_compiled_execution.py index bf00338628..4ba86c3075 100644 --- a/tests/unit/test_compiled_execution.py +++ b/tests/unit/test_compiled_execution.py @@ -53,6 +53,9 @@ from openhcs.processing.materialization import CsvOptions, MaterializationSpec from openhcs.processing.materialization.persistence import TerminalMaterializationSpec from openhcs.domains.microscopy.axes import Microscopy +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) def _runtime_environment() -> CompiledRuntimeEnvironmentPlan: @@ -278,8 +281,7 @@ def test_runtime_execution_observation_merges_into_parent_contexts(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, - group_keys=("DAPI",), + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Channel), ), MeasurementTable( name="measurements", @@ -318,8 +320,7 @@ def test_worker_runtime_observation_excludes_inherited_store_history(monkeypatch name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, - group_keys=("DAPI",), + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Channel), ) def measurement_value(object_id: int) -> RuntimeValue: diff --git a/tests/unit/test_debug_runtime.py b/tests/unit/test_debug_runtime.py index 0c51d22d04..a1c04c1d7a 100644 --- a/tests/unit/test_debug_runtime.py +++ b/tests/unit/test_debug_runtime.py @@ -14,7 +14,10 @@ from openhcs.constants.constants import MEMORY_TYPE_NUMPY from openhcs.core.callable_contract import CallableContract, CallableDeclaration -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.debug import ( NoOpDebugExecutionPolicy, @@ -307,12 +310,12 @@ def compiled_invocation( spec=spec, storage_plan=plan, projection=ArtifactInputProjectionPlan( - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=plan.producer_group_scope(), + invocation_scope=UngroupedScope(), + producer_selection_scope=plan.group_scope, component_scopes=( () - if plan.producer_group_scope().is_ungrouped - else (plan.producer_group_scope(),) + if plan.group_scope.is_ungrouped + else (plan.group_scope,) ), ), ) @@ -334,7 +337,7 @@ def execution_plan(*, artifact_inputs, artifact_outputs) -> CompiledStepPlan: input_memory_type=MEMORY_TYPE_NUMPY, source_binding_plan=CompiledSourceBindingPlan(), variable_components=(), - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), compiled_function_pattern=SimpleNamespace(is_grouped=False), artifact_inputs=artifact_inputs, artifact_outputs=artifact_outputs, @@ -1501,8 +1504,7 @@ def test_debug_snapshot_round_trips_invocation_parameters_and_artifact_identity( name="objects", path="/tmp/objects.json", artifact_type=ObjectLabelsArtifactType, - group_component=Microscopy.Well, - group_keys=("A01",), + group_scope=ComponentGroupScope.of(("A01",), component=Microscopy.Well), ) event = DebugEvent( event_type=AfterInvocationDebugBoundary, @@ -1546,8 +1548,7 @@ def test_paused_worker_runtime_inspection_projects_runtime_value_store(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, - group_keys=("DAPI",), + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Channel), ), measurements, axis_id=DebugRuntimeFixture.AXIS_ID, diff --git a/tests/unit/test_derived_role_publication_435.py b/tests/unit/test_derived_role_publication_435.py index 1abc08bf19..c86d287452 100644 --- a/tests/unit/test_derived_role_publication_435.py +++ b/tests/unit/test_derived_role_publication_435.py @@ -92,7 +92,7 @@ def test_saved_roles_publish_once_per_persisted_occurrence(tmp_path, scenario, a components.update(channel="1", z_index="0") plan.variable_components = (Microscopy.ZIndex,) plan.group_by = Microscopy.Channel - plan.execution_group_scope = ComponentGroupScope.from_raw( + plan.execution_group_scope = ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) source_planes = [] @@ -140,7 +140,7 @@ def test_saved_roles_publish_once_per_persisted_occurrence(tmp_path, scenario, a ) if aggregate: output_plan = replace( - output_plan, group_keys=("1",), group_component=Microscopy.Channel, + output_plan, group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), variable_components=(Microscopy.ZIndex,), ) plan.artifact_outputs[output_plan.ref()] = output_plan diff --git a/tests/unit/test_funcstep_contract_validator.py b/tests/unit/test_funcstep_contract_validator.py index d6ea186248..1449833d64 100644 --- a/tests/unit/test_funcstep_contract_validator.py +++ b/tests/unit/test_funcstep_contract_validator.py @@ -16,7 +16,10 @@ from openhcs.core.artifacts.kinds import ImageArtifactType from openhcs.core.callable_contract import CallableContract, CallableDeclaration from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.function_patterns import ( CompiledFunctionPattern, FunctionPatternSyntax, @@ -123,11 +126,11 @@ def _compiled_pattern_with_exact_input_edges( storage_plan = next( plan for plan in input_plans.values() if plan.ref() == spec.ref() ) - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope invocation_scope = ( producer_scope if compiled.is_grouped - else ComponentGroupScope.ungrouped() + else UngroupedScope() ) edges.append( InvocationArtifactInputEdgePlan( @@ -672,8 +675,7 @@ def combine(image): name=name, path=f"/memory/{name}.pkl", artifact_type=ImageArtifactType, - group_keys=(key,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((key,), component=Microscopy.Channel), paths_by_group={ key: f"/memory/{name}.pkl", }, @@ -688,7 +690,7 @@ def combine(image): variable_components=(), group_by=Microscopy.Channel, artifact_inputs=input_plans, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), @@ -718,8 +720,7 @@ def process_right(image): name=name, path=f"/memory/{name}.pkl", artifact_type=ImageArtifactType, - group_keys=(key,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((key,), component=Microscopy.Channel), paths_by_group={key: f"/memory/{name}.pkl"}, ) for name, key in (("left", "1"), ("right", "2")) @@ -732,7 +733,7 @@ def process_right(image): variable_components=(), group_by=Microscopy.Channel, artifact_inputs=input_plans, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), diff --git a/tests/unit/test_function_artifact_materialization.py b/tests/unit/test_function_artifact_materialization.py index 55c8cd4439..43df939d7c 100644 --- a/tests/unit/test_function_artifact_materialization.py +++ b/tests/unit/test_function_artifact_materialization.py @@ -45,7 +45,9 @@ ) from openhcs.core.component_group_scope import ( ComponentGroupScope, + DynamicGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.components.parser_metaprogramming import FilenameParseResult from openhcs.core.config import TiffConfig @@ -414,10 +416,10 @@ def _plan( variable_components = tuple(variable_components) if execution_group_value is None: execution_group_value = group_by_value - execution_group_scope = ComponentGroupScope.ungrouped() + execution_group_scope = UngroupedScope() if execution_group_value is not None: execution_group_component = AxisFamily.active().named(execution_group_value) - execution_group_scope = ComponentGroupScope.dynamic(execution_group_component) + execution_group_scope = DynamicGroupScope(execution_group_component) return CompiledStepPlan( step_index=6, step_name="measure", @@ -1074,8 +1076,7 @@ def test_materialize_artifact_outputs_uses_runtime_record_identity_not_final_pat path="/analysis/A01_measurements_step7.csv", artifact_type=MeasurementsArtifactType, materialization=csv_only(), - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/analysis/A01_measurements_step7.csv"}, ) runtime_output_plan = output_plan.for_group("1") @@ -1467,8 +1468,7 @@ def test_grouped_special_output_retains_group_coordinate_in_aggregate_name(parse name="cell_counts", path="/memory/cell_counts.pkl", artifact_type=SpecialArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), materialization=csv_only(), ) context = _context(FileManagerStub()) @@ -2105,8 +2105,7 @@ def test_materialize_artifact_outputs_unions_measurement_subject_records( path="/memory/A01_measurements_step7.pkl", artifact_type=MeasurementsArtifactType, materialization=csv_only(), - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/A01_w1_measurements_step7.pkl"}, ) runtime_output_plan = output_plan.for_group("1") @@ -2171,8 +2170,7 @@ def test_fixed_component_scopes_materialize_distinct_measurement_paths() -> None name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/measurements.pkl"}, materialization=csv_only(), ) @@ -2253,8 +2251,7 @@ def test_artifact_name_materialization_ignores_incomplete_source_identity() -> N name="SavedImage", path="/memory/SavedImage.pkl", artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": "/memory/SavedImage.pkl"}, materialization=MaterializationSpec( ImageFileOptions( @@ -2302,8 +2299,7 @@ def test_fixed_scope_preserves_distinct_source_group_identity() -> None: name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/measurements.pkl"}, materialization=csv_only(), ) @@ -2357,8 +2353,7 @@ def test_fixed_component_scope_rejects_unresolved_filename_identity() -> None: name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/measurements.pkl"}, materialization=csv_only(), ) @@ -2466,8 +2461,7 @@ def test_materialize_artifact_outputs_uses_actual_group_records(monkeypatch): name="measurements", path="/memory/A01_measurements_step7.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/A01_w1_measurements_step7.pkl", "2": "/memory/A01_w2_measurements_step7.pkl", @@ -2586,8 +2580,7 @@ def test_overlay_and_save_image_materialization_use_source_address_owner( name=artifact_name, path=f"/memory/A01_{artifact_name}_step7.pkl", artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={None: f"/memory/A01_{artifact_name}_step7.pkl"}, materialization=TerminalMaterializationSpec( ImageFileOptions(filename_suffix=".tif") @@ -2632,8 +2625,7 @@ def test_scalar_image_materialization_rejects_conflicting_duplicate_payloads(): name="Overlay", path="/memory/A01_Overlay_step7.pkl", artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={None: "/memory/A01_Overlay_step7.pkl"}, materialization=TerminalMaterializationSpec( ImageFileOptions(filename_suffix=".tif") @@ -2660,8 +2652,7 @@ def test_actual_materialization_records_uses_dynamic_runtime_groups(): name="segmentation_masks", path="/memory/A01_segmentation_masks_step7.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={None: "/memory/A01_segmentation_masks_step7.pkl"}, ) group_one = output_plan.for_group("1") @@ -2704,8 +2695,7 @@ def test_materialize_artifact_outputs_uses_group_measurement_artifact_identity( name="measurements", path="/memory/A01_measurements_step7.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group={ "1": "/memory/A01_s001_measurements_step7.pkl", "2": "/memory/A01_s002_measurements_step7.pkl", @@ -2794,8 +2784,7 @@ def test_materialize_artifact_outputs_uses_grouped_scope_not_record_location( name="measurements", path="/memory/A01_measurements_step7.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={ "2": "/memory/A01_w2_measurements_step7.pkl", }, @@ -2845,8 +2834,7 @@ def test_materialize_artifact_outputs_uses_null_component_group_identity_for_str name="Nuclei", path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={ "2": "/memory/channel2_Nuclei.pkl", }, @@ -3151,8 +3139,7 @@ def test_tile_positions_runtime_materializes_native_json_without_changing_payloa path=str(tmp_path / "runtime" / "positions.pkl"), artifact_type=spec.artifact_type, materialization=spec.materialization, - group_component=Microscopy.Channel if group_key is not None else None, - group_keys=(group_key,), + group_scope=ComponentGroupScope.of((group_key,), component=Microscopy.Channel if group_key is not None else None), ) ensure_storage_registry() filemanager = FileManager(dict(storage_registry)) @@ -3510,8 +3497,7 @@ def test_materialize_artifact_outputs_uses_runtime_plane_group_identity( path="/memory/AdjacentImage.pkl", artifact_type=ImageArtifactType, materialization=csv_only(), - group_keys=("11",), - group_component=Microscopy.Timepoint, + group_scope=ComponentGroupScope.of(("11",), component=Microscopy.Timepoint), paths_by_group={ "11": "/memory/A01_t11_AdjacentImage_step7.pkl", }, diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index a194b21732..5350f15cf7 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -30,7 +30,9 @@ from openhcs.core.artifacts.specs import ArtifactSpec from openhcs.core.component_group_scope import ( ComponentGroupScope, + DynamicGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.component_set import ComponentSet from openhcs.core.compiled_step_plan import CompiledStepPlan @@ -186,7 +188,7 @@ class CoreExecutionRequest: artifact_inputs: Mapping[ArtifactSpecRef, ArtifactInputPlan] artifact_outputs: Mapping[ArtifactSpecRef, ArtifactOutputPlan] group_key: str = "default" - execution_group_scope: ComponentGroupScope = ComponentGroupScope.ungrouped() + execution_group_scope: ComponentGroupScope = UngroupedScope() processing_contract: ProcessingContract = Pure3DContract runtime_plane_count: int = 1 @@ -208,7 +210,7 @@ def _execute_function_core(request: CoreExecutionRequest): compiled_pattern, artifact_inputs=request.artifact_inputs, relation_source_scopes={ - plan.ref(): plan.producer_group_scope() + plan.ref(): plan.group_scope for plan in request.artifact_inputs.values() }, execution_group_scope=request.execution_group_scope, @@ -1095,8 +1097,7 @@ def segment(image): name="segmentation_masks", path="/memory/A01_w2_segmentation_masks.pkl", artifact_type=ObjectLabelsArtifactType, - group_component=Microscopy.Channel, - group_keys=("2",), + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={ "2": "/memory/A01_w2_segmentation_masks.pkl", }, @@ -1104,7 +1105,7 @@ def segment(image): ) }, group_key="2", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), ) ) @@ -1160,8 +1161,7 @@ def segment(image): name="segmentation_masks", path="/memory/A01_segmentation_masks.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={ None: "/memory/A01_segmentation_masks.pkl", }, @@ -1169,7 +1169,7 @@ def segment(image): ) }, group_key="2", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), ) ) @@ -1579,8 +1579,7 @@ def analyze(image): path="/memory/A01_measurements.pkl", artifact_type=MeasurementsArtifactType, relations=measurement_spec.relations, - group_component=Microscopy.Site, - group_keys=("1", "2"), + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group={ "1": "/memory/A01_s1_measurements.pkl", "2": "/memory/A01_s2_measurements.pkl", @@ -1589,7 +1588,7 @@ def analyze(image): ) }, group_key="2", - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Site, ), @@ -1752,7 +1751,7 @@ def rgb_output(image): axis_id="A01", output_memory_type=MEMORY_TYPE_NUMPY, artifact_inputs={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), artifact_outputs={ output_plan.ref(): output_plan, }, @@ -1801,7 +1800,7 @@ def test_pattern_group_runtime_uses_declared_output_slice_cardinality(): axis_id="A01", output_memory_type=MEMORY_TYPE_NUMPY, artifact_inputs={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), artifact_outputs={}, ), component_value=None, @@ -1852,7 +1851,7 @@ def test_pattern_group_runtime_projects_nominal_object_label_stack(): axis_id="A01", output_memory_type=MEMORY_TYPE_NUMPY, artifact_inputs={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), artifact_outputs={}, ), component_value=None, diff --git a/tests/unit/test_function_core_executor_ownership.py b/tests/unit/test_function_core_executor_ownership.py index a993ab6f10..4012c31194 100644 --- a/tests/unit/test_function_core_executor_ownership.py +++ b/tests/unit/test_function_core_executor_ownership.py @@ -12,7 +12,10 @@ from openhcs.core.artifacts.specs import ArtifactSpec, ArtifactSpecCollection from openhcs.core.artifacts.kinds import MetadataArtifactType from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.function_patterns import ( CompiledMetadataArtifactInputEdgePlan, InvocationArtifactInputEdgePlan, @@ -67,7 +70,7 @@ def _executor(): axis_id="A01", input_memory_type="numpy", output_memory_type="numpy", - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), source_binding_plan=CompiledSourceBindingPlan(), ) artifacts = ({edge.key: edge for edge in edges}, {}) diff --git a/tests/unit/test_function_execution_lifecycle_anchors.py b/tests/unit/test_function_execution_lifecycle_anchors.py index 33d7d68b07..59ddc4f08d 100644 --- a/tests/unit/test_function_execution_lifecycle_anchors.py +++ b/tests/unit/test_function_execution_lifecycle_anchors.py @@ -12,7 +12,10 @@ from openhcs.core.artifacts.plans import ArtifactInputPlan from openhcs.core.artifacts.specs import ArtifactSpec from openhcs.core.artifacts.kinds import MeasurementsArtifactType -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.component_set import ComponentSet from openhcs.core.function_patterns import ( RuntimeInvocationDomain, @@ -98,8 +101,7 @@ def consume_measurements(image, *, runtime): artifact_type=MeasurementsArtifactType, source_step_id=7, source_step_scope_id="measurement-producer", - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/measurements_w2.pkl"}, ) compiled_pattern = compile_function_pattern( @@ -113,9 +115,9 @@ def consume_measurements(image, *, runtime): compiled_pattern, artifact_inputs={measurements_plan.ref(): measurements_plan}, relation_source_scopes={ - measurements.ref(): measurements_plan.producer_group_scope(), + measurements.ref(): measurements_plan.group_scope, }, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -136,7 +138,7 @@ def consume_measurements(image, *, runtime): bindings=(NamedSourceBinding(alias="OrigDNA"),) ), execution_group_value="channel", - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -179,7 +181,7 @@ def test_source_anchored_group_uses_exact_main_flow_producer_manifest() -> None: ), source_binding_plan=CompiledSourceBindingPlan(), execution_group_value=None, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), compiled_function_pattern=compiled_pattern, ) diff --git a/tests/unit/test_function_output_manifest_input_scope.py b/tests/unit/test_function_output_manifest_input_scope.py index db5091d28a..f19fee8830 100644 --- a/tests/unit/test_function_output_manifest_input_scope.py +++ b/tests/unit/test_function_output_manifest_input_scope.py @@ -178,7 +178,7 @@ def consume(image, *, labels=None, illumination_function=None): artifact_type=spec.artifact_type, source_step_scope_id=source_scope_id, ) - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope edges.append( InvocationArtifactInputEdgePlan( key=InvocationArtifactInputProjectionKey( @@ -417,7 +417,7 @@ def calculate_illumination(image): artifact_type=source.artifact_type, source_step_scope_id="color_to_gray", ) - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope invocation = invocation.with_artifact_input_edges( ( InvocationArtifactInputEdgePlan( diff --git a/tests/unit/test_function_patterns.py b/tests/unit/test_function_patterns.py index 09b99e894f..a91019b305 100644 --- a/tests/unit/test_function_patterns.py +++ b/tests/unit/test_function_patterns.py @@ -25,7 +25,10 @@ GroupLineageSourceRelation, ) from openhcs.core.callable_contract import CallableContract, FunctionStepExecutionScope -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.config import DtypeConfig from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_contract_metadata import FunctionContractAttribute @@ -88,7 +91,7 @@ def exact_input_edge( storage_plan: ArtifactInputPlan, parameter_name: str | None, ) -> InvocationArtifactInputEdgePlan: - producer_scope = storage_plan.producer_group_scope() + producer_scope = storage_plan.group_scope return InvocationArtifactInputEdgePlan( key=InvocationArtifactInputProjectionKey( invocation_key=invocation.key, @@ -751,8 +754,7 @@ def measure_channels(image, *, runtime): path="/memory/measurements.pkl", artifact_type=measurements.artifact_type, relations=measurements.relations, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), group_scope_sources_by_group={ "1": (blue.ref(),), "2": (green.ref(),), @@ -784,7 +786,7 @@ def measure_channels(image, *, runtime): ) ) group = replace(compiled.default_group, invocations=(invocation,)) - execution_scope = ComponentGroupScope.from_raw( + execution_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -817,7 +819,7 @@ def measure_channels(image, *, runtime): storage_plan=storage_plans[edge.spec.ref()], projection=ArtifactInputProjectionPlan( invocation_scope=execution_scope, - producer_selection_scope=storage_plans[edge.spec.ref()].producer_group_scope(), + producer_selection_scope=storage_plans[edge.spec.ref()].group_scope, ), ) for edge in invocation.artifact_input_edges @@ -875,8 +877,7 @@ def measure_with_aggregate(image, *, runtime): path="/memory/blue.pkl", artifact_type=scoped.artifact_type, relations=scoped.relations, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), group_scope_sources_by_group={"1": (source.ref(),)}, ) invocation = compile_function_pattern( @@ -901,7 +902,7 @@ def measure_with_aggregate(image, *, runtime): invocation = invocation.with_artifact_input_edges((edge,)) active_outputs = invocation.output_plans_for_component( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), @@ -954,8 +955,7 @@ def measure_green(image, *, runtime): path="/memory/measurements.pkl", artifact_type=blue_measurements.artifact_type, relations=(*blue_measurements.relations, *green_measurements.relations), - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), group_scope_sources_by_group={ "1": (blue.ref(),), "2": (green.ref(),), @@ -966,7 +966,7 @@ def measure_green(image, *, runtime): {}, {output_plan.ref(): output_plan}, ).default_group.invocations - execution_scope = ComponentGroupScope.from_raw( + execution_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -1204,16 +1204,14 @@ def consume_cross_channel(image, *, runtime): name="cross_channel", path="/memory/cross_channel.pkl", artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": "/memory/cross_channel__3.pkl"}, ) current_channel_plan = ArtifactInputPlan( name="current_channel", path="/memory/current_channel.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/current_channel__1.pkl"}, ) compiled = compile_function_pattern( @@ -1270,16 +1268,14 @@ def runtime_recorded_outputs(image, *, runtime): name="Cells", path="/memory/Cells.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": "/memory/w3_Cells.pkl"}, ) measurement_output = ArtifactOutputPlan( name="Measurements", path="/memory/Measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/w1_Measurements.pkl"}, ) compiled = compile_function_pattern( @@ -1822,10 +1818,10 @@ def consume(image, *, runtime): group = replace(compiled.default_group, invocations=(invocation,)) assert invocation.adapter_manages_artifact_inputs assert invocation.contract.accepts_implicit_main_flow_input - assert group.stored_primary_input_edges_for_component(ComponentGroupScope.ungrouped(), None, source_bindings=CompiledSourceBindingPlan()) is None + assert group.stored_primary_input_edges_for_component(UngroupedScope(), None, source_bindings=CompiledSourceBindingPlan()) is None primary = replace(edge, main_flow_projection=MainFlowInputProjection.COMPLETE_PAYLOAD) group = replace(group, invocations=(invocation.with_artifact_input_edges((primary,)),)) - assert group.stored_primary_input_edges_for_component(ComponentGroupScope.ungrouped(), None, source_bindings=CompiledSourceBindingPlan()) == (primary,) + assert group.stored_primary_input_edges_for_component(UngroupedScope(), None, source_bindings=CompiledSourceBindingPlan()) == (primary,) def test_table_context_cannot_stand_in_for_an_image_cohort(): @@ -1844,4 +1840,4 @@ def consume(*, runtime): parameter_name=None, ) group = replace(compiled.default_group, invocations=(invocation.with_artifact_input_edges((edge,)),)) - assert group.stored_primary_input_edges_for_component(ComponentGroupScope.ungrouped(), None, source_bindings=CompiledSourceBindingPlan()) is None + assert group.stored_primary_input_edges_for_component(UngroupedScope(), None, source_bindings=CompiledSourceBindingPlan()) is None diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index c2615d7a08..9aaf55246d 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -39,6 +39,8 @@ from openhcs.core.config import GlobalPipelineConfig from openhcs.core.component_group_scope import ( ComponentGroupScope, + DynamicGroupScope, + UngroupedScope, ) from openhcs.core.component_set import ComponentSet from openhcs.core.function_patterns import ( @@ -1475,7 +1477,7 @@ def test_step_output_anchor_uses_compiler_owned_component_scope() -> None: source_step_index=4, source_step_scope_id="object_to_image", ), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("0",), component=Microscopy.Channel, ), @@ -1521,7 +1523,7 @@ def identify_secondary(image): ), source_binding_plan=CompiledSourceBindingPlan(), execution_group_value="channel", - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), @@ -1573,9 +1575,9 @@ def identify_primary_objects(image, *, runtime): compiled_pattern, artifact_inputs={runtime_plan.ref(): runtime_plan}, relation_source_scopes={ - runtime_image.ref(): runtime_plan.producer_group_scope(), + runtime_image.ref(): runtime_plan.group_scope, }, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -1595,7 +1597,7 @@ def identify_primary_objects(image, *, runtime): ), source_binding_plan=CompiledSourceBindingPlan(), execution_group_value="channel", - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -1668,7 +1670,7 @@ def test_step_output_anchor_resolves_dynamic_component_scope_from_patterns() -> source_step_index=4, source_step_scope_id="crop", ), - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), compiled_function_pattern=compile_function_pattern( admitted_graph(lambda image: image), {}, @@ -1701,7 +1703,7 @@ def test_step_output_anchor_resolves_dynamic_component_scope_from_patterns() -> def test_source_anchor_uses_compiler_owned_static_component_scope() -> None: plan = SimpleNamespace( main_input_dependency=StepInputDependency.pipeline_start(), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -1772,8 +1774,7 @@ def measure_source_set(image): name="SourceSetMeasurements", path="/memory/SourceSetMeasurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), relations=source_set_measurements.relations, ) @@ -1783,7 +1784,7 @@ def measure_source_set(image): step_name="SourceBoundAnchorFilter", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), @@ -1903,9 +1904,9 @@ def save_image(image, *, image_to_save: np.ndarray): compiled_pattern, artifact_inputs={runtime_plan.ref(): runtime_plan}, relation_source_scopes={ - runtime_image.ref(): runtime_plan.producer_group_scope(), + runtime_image.ref(): runtime_plan.group_scope, }, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ), @@ -1949,7 +1950,7 @@ def save_image(image, *, image_to_save: np.ndarray): step_name="SaveImages", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ), @@ -2007,7 +2008,7 @@ def exact_source_input(image): step_name="ExactSourceInput", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -2061,7 +2062,7 @@ def exact_source_input(image): step_name="ExactSourceInput", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -2114,7 +2115,7 @@ def test_pipeline_start_anchors_project_raw_selectors_to_semantic_groups() -> No axis_id="A01", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=CompiledSourceBindingPlan(bindings=bindings), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("MCP_DNA", "MCP_AGP"), component=Microscopy.Channel, ), @@ -2298,7 +2299,7 @@ def source_bound_module(image, *, runtime): main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, execution_group_value=None, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), compiled_function_pattern=compiled_pattern, ) pattern_filter = _anchor_executor( @@ -2336,7 +2337,7 @@ def test_default_callable_runtime_scope_projects_bindings_to_selected_group() -> context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=source_binding_plan, compiled_function_pattern=compile_function_pattern( admitted_graph(lambda image: image), @@ -2381,7 +2382,7 @@ def test_dict_callable_runtime_scope_projects_bindings_to_selected_group() -> No context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=source_binding_plan, compiled_function_pattern=compiled_pattern, ), @@ -2424,7 +2425,7 @@ def test_grouped_runtime_adapter_receives_component_selected_source_bindings() - step_index=0, step_name="consume channel source", axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=source_binding_plan, compiled_function_pattern=compiled_pattern, variable_components=(Microscopy.Site,), @@ -2500,8 +2501,7 @@ def test_runtime_invocation_uses_only_active_source_bound_main_flow_edges( name=spec.name, path=f"/memory/{spec.name}.pkl", artifact_type=spec.artifact_type, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), paths_by_group={channel: f"/memory/{spec.name}__{channel}.pkl"}, relations=spec.relations, ) @@ -2546,7 +2546,7 @@ def consume_channel_source(image): step_index=0, step_name="consume channel source", axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=source_binding_plan, input_memory_type="numpy", variable_components=(Microscopy.Site,), @@ -2654,8 +2654,8 @@ def consume_original(image, *, runtime): spec=source_spec, storage_plan=storage, projection=ArtifactInputProjectionPlan( - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=storage.producer_group_scope(), + invocation_scope=UngroupedScope(), + producer_selection_scope=storage.group_scope, ), main_flow_projection=MainFlowInputProjection.COMPLETE_PAYLOAD, ), @@ -2711,16 +2711,14 @@ def test_runtime_chain_skips_adapter_invocation_without_component_outputs( name=first_spec.name, path="/memory/FirstLabels.pkl", artifact_type=first_spec.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/FirstLabels_1.pkl"}, ) second_plan = ArtifactOutputPlan( name=second_spec.name, path="/memory/SecondLabels.pkl", artifact_type=second_spec.artifact_type, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/SecondLabels_2.pkl"}, ) @@ -2771,7 +2769,7 @@ def record_second_labels(image, *, runtime): step_index=0, step_scope_id="pipeline::step_0", step_name="record labels", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=CompiledSourceBindingPlan(), input_memory_type="numpy", variable_components=(Microscopy.Site,), @@ -2850,7 +2848,7 @@ def exact_source_input(image): context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), source_binding_plan=source_binding_plan, compiled_function_pattern=compiled_pattern, ), @@ -2928,7 +2926,7 @@ def main_flow_with_auxiliary_source(image): context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=source_binding_plan, compiled_function_pattern=compiled_pattern, ), @@ -2971,7 +2969,7 @@ def save_main_flow_with_source_prefix(image): context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=source_binding_plan, variable_components=(Microscopy.Site,), compiled_function_pattern=compiled_pattern, @@ -3009,7 +3007,7 @@ def test_payload_provenance_outranks_unrelated_artifact_execution_group() -> Non context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -3084,7 +3082,7 @@ def consume_channel_stack(image): context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), source_binding_plan=source_binding_plan, variable_components=(Microscopy.Channel,), ), @@ -3165,7 +3163,7 @@ def consume_selected_channels(image): context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), source_binding_plan=source_binding_plan, compiled_function_pattern=compiled_pattern, ), @@ -3234,7 +3232,7 @@ def measure_neurites(image, pixel_size=1.0): context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="R04C09", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), source_binding_plan=source_binding_plan, compiled_function_pattern=compiled_pattern, ), @@ -3370,7 +3368,7 @@ def test_runtime_plane_count_comes_from_loaded_slices_not_dispatch_groups() -> N plan = SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=CompiledSourceBindingPlan(), artifact_inputs={}, artifact_outputs={}, @@ -3447,16 +3445,14 @@ def test_grouped_main_flow_context_uses_component_selected_output_plan() -> None name=corrected_stain_1_spec.name, path="/memory/CorrectedStain1.pkl", artifact_type=corrected_stain_1_spec.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/CorrectedStain1_1.pkl"}, ) corrected_stain_2 = ArtifactOutputPlan( name=corrected_stain_2_spec.name, path="/memory/CorrectedStain2.pkl", artifact_type=corrected_stain_2_spec.artifact_type, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/CorrectedStain2_2.pkl"}, ) plan = SimpleNamespace( @@ -3464,7 +3460,7 @@ def test_grouped_main_flow_context_uses_component_selected_output_plan() -> None artifact_outputs={ plan.ref(): plan for plan in (corrected_stain_1, corrected_stain_2) }, - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), source_binding_plan=CompiledSourceBindingPlan(), ) @@ -3534,7 +3530,7 @@ def record_mixed_outputs(image, *, runtime): execution_plan=SimpleNamespace( artifact_inputs={}, artifact_outputs=output_plans, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), source_binding_plan=CompiledSourceBindingPlan(), ), compiled_group=compiled_group, @@ -3562,22 +3558,20 @@ def test_component_output_selection_keeps_distinct_axes_with_equal_keys() -> Non name="Stain1", path="/memory/Stain1.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Stain1_1.pkl"}, ) channel_2 = ArtifactOutputPlan( name="Stain2", path="/memory/Stain2.pkl", artifact_type=ImageArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/Stain2_2.pkl"}, ) plan = SimpleNamespace( artifact_inputs={}, artifact_outputs={plan.ref(): plan for plan in (channel_1, channel_2)}, - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), ) selected = PatternGroupExecutionScope._select_output_plans_for_component( @@ -3653,7 +3647,7 @@ def test_component_artifact_plans_reject_malformed_exact_plan_maps() -> None: step_plan = SimpleNamespace( artifact_inputs=invalid_inputs, artifact_outputs=invalid_outputs, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), ) with pytest.raises(error_type, match=message): request = PatternGroupExecutionRequest( @@ -3684,7 +3678,7 @@ def test_grouped_runtime_scope_preserves_empty_source_binding_plan() -> None: context=SimpleNamespace(), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), source_binding_plan=source_binding_plan, ), compiled_group=compiled_pattern.default_group, @@ -3742,7 +3736,7 @@ def test_grouped_runtime_source_expansion_uses_scoped_bindings( ), execution_plan=SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, compiled_function_pattern=compiled_pattern, @@ -4004,7 +3998,7 @@ def test_empty_source_binding_plan_does_not_filter_runtime_inputs() -> None: ) plan = SimpleNamespace( axis_id="A01", - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=CompiledSourceBindingPlan(), ) @@ -5685,7 +5679,7 @@ def save_batch( step_name="ExplicitIdentity", pipeline_position=0, step_scope_id="explicit-identity", - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), artifact_outputs=output_plans, ), pattern_group_info="A01_s{iii}_w1_z001_t001.tif", diff --git a/tests/unit/test_function_step_execution_plan.py b/tests/unit/test_function_step_execution_plan.py index e1597e73c7..2190a33372 100644 --- a/tests/unit/test_function_step_execution_plan.py +++ b/tests/unit/test_function_step_execution_plan.py @@ -16,6 +16,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.artifacts.input_projection import ArtifactInputProjectionPlan from openhcs.core.artifacts.plans import ArtifactInputPlan, ArtifactOutputPlan @@ -117,7 +118,7 @@ def test_compiled_step_plan_is_the_runtime_plan_owner(): from openhcs.core.steps.function_execution import FunctionStepExecutor compiled_plan = _compiled_plan( - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Site, ) @@ -372,7 +373,7 @@ def second(image): axis_id="A01", step_name="dict-none", execution_group_value=None, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), compiled_function_pattern=compile_function_pattern( admitted_graph({"1": first, "2": second}), {}, @@ -419,8 +420,7 @@ def test_component_artifact_plan_selection_merges_global_and_group_outputs(): name="measurements", path="/tmp/measurements/A01", artifact_type=MeasurementsArtifactType, - group_keys=("A01",), - group_component=Microscopy.Well, + group_scope=ComponentGroupScope.of(("A01",), component=Microscopy.Well), paths_by_group={"A01": "/tmp/measurements/A01"}, ) @@ -429,7 +429,7 @@ def test_component_artifact_plan_selection_merges_global_and_group_outputs(): global_output.ref(): global_output, grouped_output.ref(): grouped_output, }, - ComponentGroupScope(("A01",), component=Microscopy.Well), + ComponentGroupScope.of(("A01",), component=Microscopy.Well), "A01", ) @@ -444,14 +444,13 @@ def test_component_artifact_plan_selection_omits_unscoped_outputs_for_missing_gr name="objects", path="/tmp/objects", artifact_type=ObjectLabelsArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": "/tmp/objects_w3"}, ) selected = PatternGroupExecutionScope._select_output_plans_for_component( {output.ref(): output}, - ComponentGroupScope(("1", "3"), component=Microscopy.Channel), + ComponentGroupScope.of(("1", "3"), component=Microscopy.Channel), "1", ) @@ -463,8 +462,7 @@ def test_default_invocation_keeps_compiled_grouped_output_plan(): name="measurements", path="/tmp/measurements", artifact_type=MeasurementsArtifactType, - group_keys=("1", "3", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "3", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/tmp/w1_measurements", "3": "/tmp/w3_measurements", @@ -474,7 +472,7 @@ def test_default_invocation_keeps_compiled_grouped_output_plan(): selected = PatternGroupExecutionScope._select_output_plans_for_component( {grouped_output.ref(): grouped_output}, - ComponentGroupScope.ungrouped(), + UngroupedScope(), None, ) @@ -499,16 +497,14 @@ def publish_channels(image, *, runtime): name=channel_one_spec.name, path="/memory/channel_one.pkl", artifact_type=channel_one_spec.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/channel_one__1.pkl"}, ) channel_two = ArtifactOutputPlan( name=channel_two_spec.name, path="/memory/channel_two.pkl", artifact_type=channel_two_spec.artifact_type, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/channel_two__2.pkl"}, ) invocation = compile_function_pattern( @@ -529,7 +525,7 @@ def test_invocation_output_selection_omits_inactive_component_outputs(): selected_outputs = invocation.select_outputs( {channel_one.ref(): channel_one}, compiled_output_plans=invocation.output_plans_for_component( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), "1", @@ -666,8 +662,7 @@ def publish_derived_channels(image, *, illumination_function=None, runtime): name=spec.name, path=f"/memory/{spec.name}.pkl", artifact_type=spec.artifact_type, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), paths_by_group={channel: f"/memory/{spec.name}__{channel}.pkl"}, relations=spec.relations, ) @@ -681,7 +676,7 @@ def publish_derived_channels(image, *, illumination_function=None, runtime): input_plans, output_plans, ).default_group.invocations[0] - execution_scope = ComponentGroupScope.from_raw( + execution_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -696,7 +691,7 @@ def publish_derived_channels(image, *, illumination_function=None, runtime): storage_plan=input_plans[spec.ref()], projection=ArtifactInputProjectionPlan( invocation_scope=execution_scope, - producer_selection_scope=ComponentGroupScope.ungrouped(), + producer_selection_scope=UngroupedScope(), component_scopes=(execution_scope,), consumer_variable_components=(), ), @@ -755,7 +750,7 @@ def test_invocation_output_selection_rejects_missing_active_component_output(): invocation.select_outputs( {}, compiled_output_plans=invocation.output_plans_for_component( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), "1", @@ -769,8 +764,7 @@ def test_invocation_output_selection_rejects_active_projection_drift(): name=channel_one.name, path="/memory/drifted_channel_one.pkl", artifact_type=channel_one.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/drifted_channel_one__1.pkl"}, ) @@ -778,7 +772,7 @@ def test_invocation_output_selection_rejects_active_projection_drift(): invocation.select_outputs( {drifted_channel_one.ref(): drifted_channel_one}, compiled_output_plans=invocation.output_plans_for_component( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), "1", @@ -803,15 +797,14 @@ def apply_illumination(image, *, runtime): name="IllumStain1", path="/tmp/IllumStain1", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), ) invocation = compile_function_pattern( admitted_graph(apply_illumination), {first_input.ref(): first_input}, {}, ).default_group.invocations[0] - scope = ComponentGroupScope.from_raw( + scope = ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) diff --git a/tests/unit/test_function_step_execution_scope.py b/tests/unit/test_function_step_execution_scope.py index b7d3bb53af..45c54a004f 100644 --- a/tests/unit/test_function_step_execution_scope.py +++ b/tests/unit/test_function_step_execution_scope.py @@ -27,7 +27,10 @@ CompiledStepPlan, RuntimeArtifactMaterializationPlan, ) -from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.component_group_scope import ( + RuntimeExecutionAxisScope, + UngroupedScope, +) from openhcs.core.component_group_scope import ComponentGroupScope from openhcs.core.component_set import ComponentSet from openhcs.core.context.processing_context import ProcessingContext @@ -405,9 +408,9 @@ def _plate_step_plan( compiled_pattern, artifact_inputs=input_plans, relation_source_scopes={ - plan.ref(): plan.producer_group_scope() for plan in artifact_inputs + plan.ref(): plan.group_scope for plan in artifact_inputs }, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), ) return CompiledStepPlan( @@ -485,8 +488,7 @@ def _record_measurements( name=name, path=path, artifact_type=MeasurementsArtifactType, - group_keys=(group_key,), - group_component=group_component, + group_scope=ComponentGroupScope.of((group_key,), component=group_component), ) subject = ( MeasurementSubject(MeasurementScope.ARTIFACT, name) @@ -1042,7 +1044,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): name=measurement_spec.name, path=input_path, artifact_type=measurement_spec.artifact_type, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={None: input_path}, ), ), diff --git a/tests/unit/test_implicit_main_flow_compilation.py b/tests/unit/test_implicit_main_flow_compilation.py index 91fb138d72..75d1203de9 100644 --- a/tests/unit/test_implicit_main_flow_compilation.py +++ b/tests/unit/test_implicit_main_flow_compilation.py @@ -15,7 +15,7 @@ PipelineConfig, ) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator -from openhcs.core.pipeline.path_planner import PathPlannerGroupScope +from openhcs.core.pipeline.path_planner import ComponentGroupScope from openhcs.core.progress import set_progress_queue from openhcs.core.step_dependencies import StepInputDependency from openhcs.core.steps.function_step import FunctionStep @@ -23,6 +23,9 @@ from openhcs.processing.backends.processors.numpy_processor import percentile_normalize from openhcs.domains.microscopy.axes import Microscopy from openhcs.microscopes.imagexpress import ImageXpressHandler +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) def test_native_grouped_main_flow_compiles_into_cellprofiler_consumer(tmp_path): @@ -88,7 +91,7 @@ def test_native_grouped_main_flow_compiles_into_cellprofiler_consumer(tmp_path): source_step_index=0, source_step_scope_id=native_plan.step_scope_id, ) - assert threshold_plan.execution_group_scope == PathPlannerGroupScope.from_raw( + assert threshold_plan.execution_group_scope == ComponentGroupScope.of( ("1", "2", "4"), component=Microscopy.Channel, ) diff --git a/tests/unit/test_invocation_input_source_context_identity.py b/tests/unit/test_invocation_input_source_context_identity.py index 944d350007..e4c2c83640 100644 --- a/tests/unit/test_invocation_input_source_context_identity.py +++ b/tests/unit/test_invocation_input_source_context_identity.py @@ -14,6 +14,7 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.core.component_set import ComponentSet from openhcs.constants.constants import Backend @@ -76,7 +77,7 @@ def align_like(image: np.ndarray) -> np.ndarray: compiled, artifact_inputs=input_plans, relation_source_scopes={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), main_flow_artifacts=ArtifactSpecCollection((combined_image,)), ) @@ -124,7 +125,7 @@ def threshold_like(image: np.ndarray) -> np.ndarray: compiled, artifact_inputs={}, relation_source_scopes={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), main_flow_artifacts=ArtifactSpecCollection((cursor,)), ) @@ -170,7 +171,7 @@ def consume_both(image: np.ndarray) -> np.ndarray: compile_function_pattern(admitted_graph(consume_both), {}, {}), artifact_inputs={}, relation_source_scopes={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), main_flow_artifacts=ArtifactSpecCollection((first, second)), ) diff --git a/tests/unit/test_loaded_pattern_group_ownership.py b/tests/unit/test_loaded_pattern_group_ownership.py index 66808602ca..05c9b8974e 100644 --- a/tests/unit/test_loaded_pattern_group_ownership.py +++ b/tests/unit/test_loaded_pattern_group_ownership.py @@ -10,7 +10,11 @@ from tests.artifact_graphs import admitted_graph from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + DynamicGroupScope, + UngroupedScope, +) from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.runtime_image_values import ImagePayloadMetadata @@ -42,7 +46,7 @@ def _fixture(): step_scope_id="loaded-cohort", input_memory_type="numpy", output_memory_type="numpy", - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), variable_components=(Microscopy.Site,), source_binding_plan=CompiledSourceBindingPlan(), ) @@ -136,7 +140,7 @@ def test_initial_coordinates_stay_captured_while_plan_selectors_follow_mutation( assert loaded.runtime_plane_count == 2 assert len(loaded.matching_files) == 3 request.execution_plan.axis_id = "B02" - request.execution_plan.execution_group_scope = ComponentGroupScope.dynamic( + request.execution_plan.execution_group_scope = DynamicGroupScope( Microscopy.Site ) assert loaded.axis_scope.axis_id == "B02" @@ -410,12 +414,12 @@ def consume(image, *, runtime): pattern, artifact_inputs={input_plan.ref(): input_plan}, relation_source_scopes={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet((Microscopy.Site,)), main_flow_artifacts=ArtifactSpecCollection((source,)), ) request.execution_plan.step_index = 1 - request.execution_plan.execution_group_scope = ComponentGroupScope.ungrouped() + request.execution_plan.execution_group_scope = UngroupedScope() request.execution_plan.artifact_inputs = {input_plan.ref(): input_plan} request.execution_plan.artifact_outputs = {output_plan.ref(): output_plan} request.execution_plan.compiled_function_pattern = pattern @@ -801,7 +805,7 @@ def consume(image): {}, ), artifact_inputs={}, relation_source_scopes={}, - execution_group_scope=ComponentGroupScope.dynamic(Microscopy.Channel), + execution_group_scope=DynamicGroupScope(Microscopy.Channel), consumer_variable_components=ComponentSet((Microscopy.Site,)), main_flow_artifacts=ArtifactSpecCollection( (source,) if primary_only else (source, ArtifactSpec.input("Other", ImageArtifactType)), diff --git a/tests/unit/test_opaque_main_flow_image_domain.py b/tests/unit/test_opaque_main_flow_image_domain.py index bedfc60080..3589506938 100644 --- a/tests/unit/test_opaque_main_flow_image_domain.py +++ b/tests/unit/test_opaque_main_flow_image_domain.py @@ -9,7 +9,10 @@ from tests.artifact_graphs import admitted_graph from openhcs.constants.constants import MEMORY_TYPE_NUMPY from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.function_patterns import MainFlowInputProjection, compile_function_pattern from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis @@ -47,7 +50,7 @@ def _runtime(): variable_components=(), artifact_inputs={}, artifact_outputs={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), ), component_value=None, ) diff --git a/tests/unit/test_output_publication_ownership.py b/tests/unit/test_output_publication_ownership.py index 5d245f5812..8d3c239f3a 100644 --- a/tests/unit/test_output_publication_ownership.py +++ b/tests/unit/test_output_publication_ownership.py @@ -16,7 +16,10 @@ ImagePayloadStackComposition, ) from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.runtime_image_values import ImagePayloadMetadata @@ -48,7 +51,7 @@ def _runtime(tmp_path): variable_components=(), artifact_inputs={}, artifact_outputs={}, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), ) runtime = PatternGroupExecutionRequest(context=context, execution_plan=plan, diff --git a/tests/unit/test_path_planner_materialization.py b/tests/unit/test_path_planner_materialization.py index 705e06ee09..39a2d88d93 100644 --- a/tests/unit/test_path_planner_materialization.py +++ b/tests/unit/test_path_planner_materialization.py @@ -36,7 +36,9 @@ from openhcs.core.component_set import ComponentSet from openhcs.core.component_group_scope import ( ComponentGroupScope, + DynamicGroupScope, RuntimeExecutionAxisScope, + UngroupedScope, ) from openhcs.constants.constants import MEMORY_TYPE_NUMPY from openhcs.core.memory import numpy @@ -90,7 +92,7 @@ PathPlannerComponentScopes, PathPlannerArtifactStage, PathPlannerExecutionGroups, - PathPlannerGroupScope, + ComponentGroupScope, PathPlannerMaterializationStage, PathPlannerPathAuthority, PathPlannerStepAssemblyStage, @@ -136,7 +138,7 @@ def _execute_compiled_metadata_pattern(compiled, input_plans=None, stored_output input_memory_type=MEMORY_TYPE_NUMPY, source_binding_plan=CompiledSourceBindingPlan(), variable_components=(), - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), compiled_function_pattern=compiled, artifact_inputs=input_plans, artifact_outputs={}, @@ -222,7 +224,7 @@ def _compile_metadata_pattern(planner, pattern): input_plans = planner.artifacts.process_artifact_inputs( declarations, 3, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), EMPTY_SOURCE_BINDINGS, ComponentSet(), snapshot.name, @@ -234,7 +236,7 @@ def _compile_metadata_pattern(planner, pattern): input_plans, {}, {}, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), declarations=planner.artifacts.inject_metadata(declarations), ) return compiled, input_plans @@ -390,7 +392,7 @@ def metadata_consumer(image, grid_dimensions): runtime_input_plans = planner.artifacts.process_artifact_inputs( declarations, 3, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), execution_bindings, ComponentSet(), snapshot.name, @@ -403,12 +405,12 @@ def metadata_consumer(image, grid_dimensions): runtime_input_plans, {}, {}, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), declarations=planner.artifacts.inject_metadata(declarations), ) assert execution_bindings == CompiledSourceBindingPlan() - assert execution_group_scope == PathPlannerGroupScope.dynamic(Microscopy.Channel) + assert execution_group_scope == DynamicGroupScope(Microscopy.Channel) assert runtime_input_plans == {} assert compiled is not None (invocation,) = compiled.default_group.invocations @@ -631,7 +633,7 @@ def export_measurements(*, artifact_batch: RuntimeArtifactBatch): snapshot, 3, declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), callable_scope, execution_bindings, ) @@ -843,7 +845,7 @@ def test_artifact_output_plans_preserve_declared_kind(): ) ), sid=2, - output_groups={output.ref(): PathPlannerGroupScope.ungrouped()}, + output_groups={output.ref(): UngroupedScope()}, artifact_inputs={}, source_bindings=EMPTY_SOURCE_BINDINGS, variable_components=ComponentSet(), @@ -883,7 +885,7 @@ def produce_shared(): producer_snapshot, 2, producer_declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), producer_scope, EMPTY_SOURCE_BINDINGS, ) @@ -946,7 +948,7 @@ def consume_shared(image, shared_image, shared_labels): consumer_snapshot, 3, consumer_declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), consumer_scope, EMPTY_SOURCE_BINDINGS, ) @@ -1141,7 +1143,7 @@ def combine_sources(image, secondary=None): compiled, artifact_inputs=stored_inputs, relation_source_scopes={}, - execution_group_scope=PathPlannerGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet((Microscopy.ZIndex,)), main_flow_artifacts=ArtifactSpecCollection(source_specs), ) @@ -1222,7 +1224,7 @@ def test_implicit_native_main_flow_provenance_drives_artifact_owned_scope(): step_name="percentile_normalize", step_index=0, ) - channel_scope = PathPlannerGroupScope.from_raw( + channel_scope = ComponentGroupScope.of( ("1", "2", "4"), component=Microscopy.Channel, ) @@ -1409,16 +1411,14 @@ def test_artifact_output_source_lookup_ignores_shared_input_broadcast_projection name=green.name, path="/memory/OrigGreen.pkl", artifact_type=ImageArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ), red.ref(): ArtifactInputPlan( name=red.name, path="/memory/OrigRed.pkl", artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ), green_mask.ref(): ArtifactInputPlan( @@ -1426,8 +1426,7 @@ def test_artifact_output_source_lookup_ignores_shared_input_broadcast_projection path="/memory/CropBlue__crop_mask.pkl", artifact_type=ImageArtifactType, sidecar_role=CropMask, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ), } @@ -1568,8 +1567,7 @@ def identify(main_image, Stain1): name=aligned_input.name, path="/memory/Stain1.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), variable_components=(Microscopy.Channel,), paths_by_group={ "1": "/memory/Stain1_site_1.pkl", @@ -1608,7 +1606,7 @@ def identify(main_image, Stain1): declarations=declarations, ) - expected_channel_scope = ComponentGroupScope.from_raw( + expected_channel_scope = ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) @@ -1623,7 +1621,7 @@ def identify(main_image, Stain1): assert edge.projection.component_scope(Microscopy.Channel) == ( expected_channel_scope ) - assert edge.projection.producer_selection_scope == ComponentGroupScope.from_raw( + assert edge.projection.producer_selection_scope == ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ) @@ -1671,7 +1669,7 @@ def test_fixed_source_component_domain_survives_produced_artifact_lineage( planner.artifact_context = ArtifactDeclarationStepContext( source_bindings=source_bindings, ).with_source_declarations((source,)) - producer_scope = PathPlannerGroupScope.from_raw( + producer_scope = ComponentGroupScope.of( ("1",), component=Microscopy.Site, ) @@ -1702,7 +1700,7 @@ def test_fixed_source_component_domain_survives_produced_artifact_lineage( execution_scope=FunctionStepExecutionScope.AXIS, )[output_spec.ref()] - fixed_channel = ComponentGroupScope.from_raw( + fixed_channel = ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) @@ -1722,7 +1720,7 @@ def consume(image): ), ), ) - consumer_scope = PathPlannerGroupScope.from_raw( + consumer_scope = ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -1757,7 +1755,7 @@ def consume(image): assert input_plan.component_domain(Microscopy.Channel) == fixed_channel assert consumer_relation_scopes[input_spec.ref()] == ( - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( fixed_channel.keys, component=fixed_channel.component, ) @@ -1855,7 +1853,7 @@ def identify(image): scopes = planner.artifacts.output_groups_from_declared_relations( namespaced, - group_scope=PathPlannerGroupScope.from_raw(("1", "2"), component=Microscopy.Channel), + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), relation_source_scopes={}, consumer_variable_components=ComponentSet(), step_index=0, step_name="identify", ) @@ -1908,7 +1906,7 @@ def declared_artifact_owner(image): assert declarations.output_groups[output_ref] == {None} scopes = planner.artifacts.output_groups_from_declared_relations( namespaced, - group_scope=PathPlannerGroupScope.from_raw(("1", "2"), component=Microscopy.Channel), + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), relation_source_scopes={}, consumer_variable_components=ComponentSet(), step_index=0, step_name="correct_illumination", ) @@ -1923,8 +1921,7 @@ def test_declared_group_lineage_scopes_outputs_without_rewriting_execution(): name="Tile_of_grid", path="/memory/Tile_of_grid.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Tile_of_grid_1.pkl"}, ), ) @@ -1962,7 +1959,7 @@ def test_declared_group_lineage_scopes_outputs_without_rewriting_execution(): _resolved_step(name="FilterObjects"), 3, declarations, - PathPlannerGroupScope.from_raw(("2",), component=Microscopy.Channel), + ComponentGroupScope.of(("2",), component=Microscopy.Channel), ) filtered_tiles_ref = ArtifactSpec.output( @@ -1977,7 +1974,7 @@ def test_declared_group_lineage_scopes_outputs_without_rewriting_execution(): assert maps.outputs[measurements_ref].group_keys == ("1",) assert maps.outputs[filtered_tiles_ref].group_component is Microscopy.Channel assert maps.outputs[measurements_ref].group_component is Microscopy.Channel - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -2005,7 +2002,7 @@ def test_declared_group_lineage_uses_main_flow_scope_without_artifact_plan(): ), ), ) - group_scope = PathPlannerGroupScope.from_raw( + group_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ) @@ -2035,8 +2032,7 @@ def test_prior_main_flow_artifact_scopes_output_without_rewriting_execution(): name="CropRed", path="/memory/CropRed.pkl", artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": "/memory/CropRed_3.pkl"}, ) _record_declared_output(planner, producer) @@ -2065,14 +2061,14 @@ def test_prior_main_flow_artifact_scopes_output_without_rewriting_execution(): _resolved_step(name="IdentifyPrimaryObjects"), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("2", "3"), component=Microscopy.Channel, ), ) assert maps.inputs == {} - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( ("2", "3"), component=Microscopy.Channel, ) @@ -2107,7 +2103,7 @@ def test_dict_invocation_lineage_uses_its_non_plan_input_group_scope(): _resolved_step(name="IdentifyPrimaryObjects"), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), @@ -2128,8 +2124,7 @@ def test_measurement_output_scope_compiles_exact_cross_group_consumer_edge(): name="Tile_of_grid", path="/memory/Tile_of_grid.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Tile_of_grid_1.pkl"}, ), ) @@ -2177,13 +2172,13 @@ def test_measurement_output_scope_compiles_exact_cross_group_consumer_edge(): _resolved_step(name="MeasureObjectIntensity"), 2, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), ) - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -2226,7 +2221,7 @@ def filter_objects(main_image, MeasureObjectIntensity_3_measurements): ), ), ), - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -2242,7 +2237,7 @@ def filter_objects(main_image, MeasureObjectIntensity_3_measurements): ) edge = next(compiled.iter_invocations()).artifact_input_edges[0] - assert edge.projection.producer_selection_scope == ComponentGroupScope( + assert edge.projection.producer_selection_scope == ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -2263,8 +2258,7 @@ def test_artifact_managed_lineage_keeps_exact_named_inputs_in_one_invocation(): name=name, path=f"/memory/{name}.pkl", artifact_type=artifact_type, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), variable_components=(Microscopy.Site,), paths_by_group={channel: f"/memory/{name}_{channel}.pkl"}, ), @@ -2313,10 +2307,10 @@ def test_artifact_managed_lineage_keeps_exact_named_inputs_in_one_invocation(): _resolved_step(name="MeasureObjectSizeShape"), 3, declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), ) - assert maps.group_scope == PathPlannerGroupScope.ungrouped() + assert maps.group_scope == UngroupedScope() assert set(maps.inputs) == {spec.ref() for spec in inputs} @@ -2328,8 +2322,7 @@ def test_artifact_managed_single_source_output_retains_source_group_scope(): name="CropBlue", path="/memory/CropBlue.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/CropBlue_1.pkl"}, ), ) @@ -2366,10 +2359,10 @@ def test_artifact_managed_single_source_output_retains_source_group_scope(): _resolved_step(name="IdentifyPrimaryObjects"), 3, declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), ) - assert maps.group_scope == PathPlannerGroupScope.ungrouped() + assert maps.group_scope == UngroupedScope() nuclei_ref = ArtifactSpec.output("Nuclei", ObjectLabelsArtifactType).ref() assert maps.outputs[nuclei_ref].group_keys == ("1",) assert maps.outputs[nuclei_ref].group_component is Microscopy.Channel @@ -2383,8 +2376,7 @@ def test_declared_group_lineage_unions_compatible_source_groups(): name="OrigStain1", path="/memory/OrigStain1.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/OrigStain1_1.pkl"}, ), ) @@ -2394,8 +2386,7 @@ def test_declared_group_lineage_unions_compatible_source_groups(): name="OrigStain2", path="/memory/OrigStain2.pkl", artifact_type=ImageArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/OrigStain2_2.pkl"}, ), ) @@ -2432,7 +2423,7 @@ def test_declared_group_lineage_unions_compatible_source_groups(): _resolved_step(name="Measure"), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), @@ -2444,22 +2435,22 @@ def test_declared_group_lineage_unions_compatible_source_groups(): ).ref() assert maps.outputs[measurements_ref].group_keys == ("1", "2") assert maps.relation_source_scopes[first_ref] == ( - PathPlannerGroupScope.from_raw(("1",), component=Microscopy.Channel) + ComponentGroupScope.of(("1",), component=Microscopy.Channel) ) assert maps.relation_source_scopes[second_ref] == ( - PathPlannerGroupScope.from_raw(("2",), component=Microscopy.Channel) + ComponentGroupScope.of(("2",), component=Microscopy.Channel) ) def test_dynamic_group_scope_union_remains_dynamic(): - dynamic_scope = PathPlannerGroupScope.dynamic(Microscopy.Site) - concrete_scope = PathPlannerGroupScope.from_raw( + dynamic_scope = DynamicGroupScope(Microscopy.Site) + concrete_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ) assert ( - PathPlannerGroupScope.union_compatible((dynamic_scope, concrete_scope)) + ComponentGroupScope.union_compatible((dynamic_scope, concrete_scope)) == dynamic_scope ) @@ -2477,8 +2468,7 @@ def test_collected_lineage_outputs_use_relation_owned_group_scope(): name=name, path=f"/memory/{name}.pkl", artifact_type=artifact_type, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), variable_components=(Microscopy.Site,), paths_by_group={channel: f"/memory/{name}_{channel}.pkl"}, ), @@ -2520,10 +2510,10 @@ def test_collected_lineage_outputs_use_relation_owned_group_scope(): ), 3, declarations, - PathPlannerGroupScope.dynamic(Microscopy.Site), + DynamicGroupScope(Microscopy.Site), ) - assert maps.group_scope == PathPlannerGroupScope.dynamic(Microscopy.Site) + assert maps.group_scope == DynamicGroupScope(Microscopy.Site) measurements_ref = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, @@ -2551,8 +2541,7 @@ def test_output_lineage_uses_input_qualified_consumer_scope(): name=nuclei.name, path="/memory/Nuclei.pkl", artifact_type=nuclei.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Nuclei_1.pkl"}, ), ) @@ -2562,8 +2551,7 @@ def test_output_lineage_uses_input_qualified_consumer_scope(): name=prior_measurements.name, path="/memory/PriorMeasurements.pkl", artifact_type=prior_measurements.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/PriorMeasurements_1.pkl", "2": "/memory/PriorMeasurements_2.pkl", @@ -2590,7 +2578,7 @@ def test_output_lineage_uses_input_qualified_consumer_scope(): _resolved_step(name="Calculate"), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), @@ -2598,7 +2586,7 @@ def test_output_lineage_uses_input_qualified_consumer_scope(): output_plan = maps.outputs[result.ref()] assert maps.relation_source_scopes[prior_measurements.ref()] == ( - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -2673,7 +2661,7 @@ def measure_green(image, *, runtime): variable_components=(Microscopy.Site,), input_source=InputSource.PIPELINE_START, ) - execution_scope = PathPlannerGroupScope.from_raw( + execution_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -2741,8 +2729,7 @@ def test_real_object_measurement_preserves_selected_labels_group_scope( name="Cells", path="/memory/Cells.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), paths_by_group={"2": "/memory/Cells_2.pkl"}, ), @@ -2812,7 +2799,7 @@ def test_real_object_measurement_preserves_selected_labels_group_scope( == ObjectMeasurementSubjectRelation(labels_input.ref()).measurement_subject() ) assert selector in consumed_names - group_scope = PathPlannerGroupScope.from_raw( + group_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel ) @@ -2853,8 +2840,7 @@ def test_declared_group_lineage_cannot_rewrite_scalar_step_execution_scope(): name="MembInvertRemoveHoles", path="/memory/MembInvertRemoveHoles.pkl", artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": "/memory/MembInvertRemoveHoles_3.pkl"}, ), ) @@ -2864,8 +2850,7 @@ def test_declared_group_lineage_cannot_rewrite_scalar_step_execution_scope(): name="MonolayerMask", path="/memory/MonolayerMask.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/MonolayerMask_1.pkl"}, ), ) @@ -2911,10 +2896,10 @@ def test_declared_group_lineage_cannot_rewrite_scalar_step_execution_scope(): _resolved_step(name="MaskImage"), 3, declarations, - PathPlannerGroupScope.from_raw(("1", "3"), component=Microscopy.Channel), + ComponentGroupScope.of(("1", "3"), component=Microscopy.Channel), ) - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( ("1", "3"), component=Microscopy.Channel, ) @@ -2945,8 +2930,7 @@ def test_artifact_output_storage_scope_is_independent_of_execution_scope(): name="PriorMeasurements", path="/memory/PriorMeasurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/PriorMeasurements_1.pkl", "2": "/memory/PriorMeasurements_2.pkl", @@ -2961,8 +2945,7 @@ def test_artifact_output_storage_scope_is_independent_of_execution_scope(): name="Nuclei", path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Nuclei_1.pkl"}, producer_step_index=2, producer_step_name="IdentifyPrimaryObjects", @@ -3000,10 +2983,10 @@ def test_artifact_output_storage_scope_is_independent_of_execution_scope(): _resolved_step(name="CalculateMath"), 3, declarations, - PathPlannerGroupScope.from_raw(("3",), component=Microscopy.Channel), + ComponentGroupScope.of(("3",), component=Microscopy.Channel), ) - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ) @@ -3023,8 +3006,7 @@ def test_each_output_storage_scope_is_independent_of_execution_scope(): name="source", path="/memory/source.pkl", artifact_type=ImageArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/source_2.pkl"}, ), ) @@ -3068,17 +3050,17 @@ def test_each_output_storage_scope_is_independent_of_execution_scope(): _resolved_step(name="invocation-scoped-output"), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), ) - execution_scope = PathPlannerGroupScope.from_raw( + execution_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) - relation_scope = PathPlannerGroupScope.from_raw( + relation_scope = ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -3088,10 +3070,10 @@ def test_each_output_storage_scope_is_independent_of_execution_scope(): "ambiguous", MeasurementsArtifactType, ).ref() - assert PathPlannerGroupScope.from_output_plan(maps.outputs[scoped_ref]) == ( + assert ComponentGroupScope.from_output_plan(maps.outputs[scoped_ref]) == ( relation_scope ) - assert PathPlannerGroupScope.from_output_plan(maps.outputs[ambiguous_ref]) == ( + assert ComponentGroupScope.from_output_plan(maps.outputs[ambiguous_ref]) == ( execution_scope ) @@ -3104,8 +3086,7 @@ def test_dict_pattern_output_groups_do_not_drive_scalar_scope_narrowing(): name="source", path="/memory/source.pkl", artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": "/memory/source_3.pkl"}, ), ) @@ -3144,10 +3125,10 @@ def test_dict_pattern_output_groups_do_not_drive_scalar_scope_narrowing(): _resolved_step(name="dict_pattern"), 3, declarations, - PathPlannerGroupScope.from_raw(("1", "3"), component=Microscopy.Channel), + ComponentGroupScope.of(("1", "3"), component=Microscopy.Channel), ) - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( ("1", "3"), component=Microscopy.Channel, ) @@ -3188,7 +3169,7 @@ def test_source_binding_component_identity_narrows_declared_output_lineage(): _resolved_step(name="Watershed", source_bindings=source_bindings), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2", "3"), component=Microscopy.Channel, ), @@ -3235,11 +3216,11 @@ def test_source_binding_identity_scopes_outputs_without_execution_fanout(): _resolved_step(name="Watershed", source_bindings=source_bindings), 3, declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), source_bindings=source_bindings, ) - assert maps.group_scope == PathPlannerGroupScope.ungrouped() + assert maps.group_scope == UngroupedScope() cells_ref = ArtifactSpec.output("Cells", ObjectLabelsArtifactType).ref() assert maps.outputs[cells_ref].group_keys == ("3",) assert maps.outputs[cells_ref].group_component is Microscopy.Channel @@ -3256,8 +3237,7 @@ def test_image_object_outputs_keep_declared_image_execution_group_scope(): name="Tile_of_grid", path="/memory/Tile_of_grid.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Tile_of_grid_1.pkl"}, ), ) @@ -3301,7 +3281,7 @@ def test_image_object_outputs_keep_declared_image_execution_group_scope(): _resolved_step(name="MeasureObjectIntensity"), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), @@ -3322,8 +3302,7 @@ def test_group_lineage_source_resolves_prior_main_flow_output_without_store_inpu name="Tile_of_grid", path="/memory/Tile_of_grid.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Tile_of_grid_1.pkl"}, ), ) @@ -3354,7 +3333,7 @@ def test_group_lineage_source_resolves_prior_main_flow_output_without_store_inpu _resolved_step(name="FilterObjects"), 3, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), @@ -3417,7 +3396,7 @@ def declared_artifact_owner(image): {}, {output_plan.ref(): output_plan}, {}, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), declarations=declarations, ) @@ -3454,7 +3433,7 @@ def filter_objects(image, *, runtime): ) input_component_scopes = PathPlannerComponentScopes( { - Microscopy.Channel: PathPlannerGroupScope.from_raw( + Microscopy.Channel: ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -3472,7 +3451,7 @@ def filter_objects(image, *, runtime): input_component_scopes, ) - assert execution_groups == PathPlannerGroupScope.from_raw( + assert execution_groups == ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -3493,7 +3472,7 @@ def filter_objects(image, *, runtime): input_component_scopes, ) - assert execution_groups == PathPlannerGroupScope.from_raw( + assert execution_groups == ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -3536,10 +3515,10 @@ def consume(image, labels: ObjectLabelValue, *, runtime): compiled, artifact_inputs=input_plans, relation_source_scopes={ - spec.ref(): input_plans[spec.ref()].producer_group_scope() + spec.ref(): input_plans[spec.ref()].group_scope for spec in (labels, measurements) }, - execution_group_scope=PathPlannerGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet(), ) @@ -3574,14 +3553,13 @@ def identify_primary_objects(*, runtime): name=nuclei.name, path="/memory/Nuclei.pkl", artifact_type=nuclei.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), component_domains=( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ), @@ -3599,7 +3577,7 @@ def identify_primary_objects(*, runtime): ) main_flow_scopes = PathPlannerComponentScopes( { - Microscopy.Channel: PathPlannerGroupScope.from_raw( + Microscopy.Channel: ComponentGroupScope.of( ("2", "3"), component=Microscopy.Channel, ) @@ -3612,7 +3590,7 @@ def identify_primary_objects(*, runtime): contracts=(CallableContract.from_callable(identify_primary_objects),), ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) @@ -3640,8 +3618,7 @@ def relate_objects(image, *, runtime): name=spec.name, path=f"/memory/{spec.name}.pkl", artifact_type=spec.artifact_type, - group_keys=(channel,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((channel,), component=Microscopy.Channel), ), ) snapshot = _resolved_step( @@ -3655,7 +3632,7 @@ def relate_objects(image, *, runtime): ) main_flow_scopes = PathPlannerComponentScopes( { - Microscopy.Channel: PathPlannerGroupScope.from_raw( + Microscopy.Channel: ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -3668,7 +3645,7 @@ def relate_objects(image, *, runtime): contracts=(CallableContract.from_callable(relate_objects),), ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -3694,8 +3671,7 @@ def measure_object_size_shape(image, *, runtime): name=comet_outline.name, path="/memory/CometOutline.pkl", artifact_type=comet_outline.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), ), ) snapshot = _resolved_step( @@ -3709,7 +3685,7 @@ def measure_object_size_shape(image, *, runtime): ) main_flow_scopes = PathPlannerComponentScopes( { - Microscopy.Channel: PathPlannerGroupScope.from_raw( + Microscopy.Channel: ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) @@ -3722,7 +3698,7 @@ def measure_object_size_shape(image, *, runtime): contracts=(CallableContract.from_callable(measure_object_size_shape),), ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) @@ -3741,7 +3717,7 @@ def test_execution_groups_resolve_non_grouped_variable_component_conflicts(): assert ( planner.execution_groups.get_execution_groups(snapshot) - == PathPlannerGroupScope.ungrouped() + == UngroupedScope() ) @@ -3766,7 +3742,7 @@ def test_non_dict_group_by_declares_dynamic_scope_without_plate_key_lookup(): source_snapshot, PathPlannerComponentScopes.empty(), ) - assert source_scope == PathPlannerGroupScope.from_raw( + assert source_scope == ComponentGroupScope.of( (None,), component=Microscopy.Channel, ) @@ -3788,7 +3764,7 @@ def test_non_dict_group_by_uses_dynamic_source_scope_for_pipeline_start(): source_snapshot, PathPlannerComponentScopes.empty(), ) - assert source_scope == PathPlannerGroupScope.from_raw( + assert source_scope == ComponentGroupScope.of( (None,), component=Microscopy.Channel, ) @@ -3813,7 +3789,7 @@ def test_dict_pattern_group_by_declares_execution_group_component(): PathPlannerComponentScopes.empty(), ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -3868,8 +3844,8 @@ def test_non_dict_group_by_preserves_explicitly_collapsed_input_axis(): planner = _artifact_planner_stub() input_scopes = PathPlannerComponentScopes( { - Microscopy.Channel: PathPlannerGroupScope.ungrouped(), - Microscopy.Site: PathPlannerGroupScope.from_raw( + Microscopy.Channel: UngroupedScope(), + Microscopy.Site: ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ), @@ -3887,7 +3863,7 @@ def test_non_dict_group_by_preserves_explicitly_collapsed_input_axis(): scope = planner.execution_groups.get_execution_groups(snapshot, input_scopes) - assert scope == PathPlannerGroupScope.ungrouped() + assert scope == UngroupedScope() def test_module_special_outputs_preserve_existing_main_flow_component_scopes(): @@ -3924,11 +3900,11 @@ def measurement_only(image, *, runtime): ) input_scopes = PathPlannerComponentScopes( { - Microscopy.Channel: PathPlannerGroupScope.from_raw( + Microscopy.Channel: ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), - Microscopy.Site: PathPlannerGroupScope.ungrouped(), + Microscopy.Site: UngroupedScope(), } ) snapshot = _resolved_step( @@ -3940,7 +3916,7 @@ def measurement_only(image, *, runtime): output_scopes = input_scopes.output_after( snapshot, - PathPlannerGroupScope.dynamic(Microscopy.Site), + DynamicGroupScope(Microscopy.Site), pattern, ) @@ -3975,11 +3951,11 @@ def image_output(image): ) input_scopes = PathPlannerComponentScopes( { - Microscopy.Channel: PathPlannerGroupScope.from_raw( + Microscopy.Channel: ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), - Microscopy.Site: PathPlannerGroupScope.ungrouped(), + Microscopy.Site: UngroupedScope(), } ) snapshot = _resolved_step( @@ -3991,13 +3967,13 @@ def image_output(image): output_scopes = input_scopes.output_after( snapshot, - PathPlannerGroupScope.dynamic(Microscopy.Site), + DynamicGroupScope(Microscopy.Site), pattern, ) assert output_scopes == PathPlannerComponentScopes( { - Microscopy.Site: PathPlannerGroupScope.dynamic(Microscopy.Site), + Microscopy.Site: DynamicGroupScope(Microscopy.Site), } ) @@ -4030,7 +4006,7 @@ def test_non_dict_group_by_namespaces_artifact_outputs_with_dynamic_component(): snapshot, 2, declarations, - PathPlannerGroupScope.from_raw( + ComponentGroupScope.of( (None,), component=Microscopy.Channel, ), @@ -4072,7 +4048,7 @@ def test_non_dict_group_by_uses_source_binding_identity_for_pipeline_start_scope PathPlannerComponentScopes.empty(), ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -4107,8 +4083,7 @@ def save_declared_payload(image): name=payload.name, path="/memory/composed.pkl", artifact_type=payload.artifact_type, - group_keys=("1", "2", "3"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2", "3"), component=Microscopy.Site), ), ) else: @@ -4136,9 +4111,9 @@ def save_declared_payload(image): ) expected = ( - PathPlannerGroupScope.dynamic(Microscopy.Channel) + DynamicGroupScope(Microscopy.Channel) if stored_payload - else PathPlannerGroupScope.from_raw(("3",), component=Microscopy.Channel) + else ComponentGroupScope.of(("3",), component=Microscopy.Channel) ) assert scope == expected assert not scope.contains_runtime_key("1") or scope.is_dynamic @@ -4199,7 +4174,7 @@ def test_main_flow_source_anchor_restricts_execution_to_its_exact_channel(): assert tuple(binding.alias for binding in execution_bindings.bindings) == ( "BF_image", ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) @@ -4229,8 +4204,7 @@ def measure(image, CropBlue): name=source.name, path="/memory/CropBlue.pkl", artifact_type=source.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), variable_components=(Microscopy.Site,), paths_by_group={ "1": "/memory/CropBlue_channel_1.pkl", @@ -4246,7 +4220,7 @@ def measure(image, CropBlue): group_by=Microscopy.Site, variable_components=(Microscopy.Channel,), ) - execution_scope = PathPlannerGroupScope.from_raw( + execution_scope = ComponentGroupScope.of( ("1", "2", "3"), component=Microscopy.Site, ) @@ -4269,21 +4243,21 @@ def measure(image, CropBlue): ) assert maps.group_scope == execution_scope - assert maps.inputs[source.ref()].producer_group_scope() == ( - ComponentGroupScope.from_raw( + assert maps.inputs[source.ref()].group_scope == ( + ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) ) - assert PathPlannerGroupScope.from_output_plan(maps.outputs[measurements.ref()]) == ( + assert ComponentGroupScope.from_output_plan(maps.outputs[measurements.ref()]) == ( execution_scope ) edge = next(compiled.iter_invocations()).artifact_input_edges[0] - assert edge.projection.invocation_scope == ComponentGroupScope.dynamic( + assert edge.projection.invocation_scope == DynamicGroupScope( Microscopy.Site ) assert edge.projection.producer_selection_scope == ( - maps.inputs[source.ref()].producer_group_scope() + maps.inputs[source.ref()].group_scope ) @@ -4368,8 +4342,7 @@ def test_runtime_artifact_input_plan_owns_relation_source_scope(): name=rgb_image.name, path="/memory/RGBImage.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2", "3"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2", "3"), component=Microscopy.Site), paths_by_group={ "1": "/memory/RGBImage_site_1.pkl", "2": "/memory/RGBImage_site_2.pkl", @@ -4388,7 +4361,7 @@ def test_runtime_artifact_input_plan_owns_relation_source_scope(): relation_scopes = planner.artifacts.relation_source_scopes_by_ref( declarations, {artifact_input.ref(): artifact_input}, - group_scope=PathPlannerGroupScope.from_raw( + group_scope=ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ), @@ -4396,7 +4369,7 @@ def test_runtime_artifact_input_plan_owns_relation_source_scope(): group_by=Microscopy.Channel, ) - assert relation_scopes[rgb_image.ref()] == PathPlannerGroupScope.from_raw( + assert relation_scopes[rgb_image.ref()] == ComponentGroupScope.of( ("1", "2", "3"), component=Microscopy.Site, ) @@ -4442,7 +4415,7 @@ def test_relation_source_scopes_rejects_malformed_exact_input_plan_maps(): planner.artifacts.relation_source_scopes_by_ref( declarations, artifact_inputs_by_ref, - group_scope=PathPlannerGroupScope.ungrouped(), + group_scope=UngroupedScope(), source_bindings=EMPTY_SOURCE_BINDINGS, group_by=None, ) @@ -4481,19 +4454,19 @@ def test_process_artifact_outputs_rejects_malformed_exact_maps(): invalid_output_groups = ( ( - {output_spec.name: PathPlannerGroupScope.ungrouped()}, + {output_spec.name: UngroupedScope()}, TypeError, "output-group maps require ArtifactSpecRef keys", ), ( {output_spec.ref(): (None,)}, TypeError, - "output-group maps require PathPlannerGroupScope values", + "output-group maps require ComponentGroupScope values", ), ( { ArtifactSpec.output("OtherOutput", ImageArtifactType).ref(): ( - PathPlannerGroupScope.ungrouped() + UngroupedScope() ) }, ValueError, @@ -4587,7 +4560,7 @@ def test_non_dict_group_by_ignores_source_binding_identity_for_other_components( PathPlannerComponentScopes.empty(), ) - assert scope == PathPlannerGroupScope.from_raw( + assert scope == ComponentGroupScope.of( (None,), component=Microscopy.Site, ) @@ -4608,7 +4581,7 @@ def test_compiled_group_by_preserves_dynamic_execution_scope(): ) artifact_maps = ArtifactPlanMaps( declarations=ArtifactGraph.empty(), - group_scope=PathPlannerGroupScope.dynamic(Microscopy.Channel), + group_scope=DynamicGroupScope(Microscopy.Channel), inputs={}, outputs={}, relation_source_scopes={}, @@ -4630,7 +4603,7 @@ def test_compiled_group_by_preserves_dynamic_execution_scope(): ) assert planner.plans[3].group_by is Microscopy.Channel - assert planner.plans[3].execution_group_scope == PathPlannerGroupScope.dynamic( + assert planner.plans[3].execution_group_scope == DynamicGroupScope( Microscopy.Channel ) assert planner.plans[3].analysis_results_dir == "/data/plate1_generated/analysis" @@ -4659,7 +4632,7 @@ def test_artifact_input_plan_requires_an_exact_producer_kind(): ), ) ), - consumer_scope=PathPlannerGroupScope.ungrouped(), + consumer_scope=UngroupedScope(), sid=2, step_name="measure", source_bindings=EMPTY_SOURCE_BINDINGS, @@ -4690,7 +4663,7 @@ def test_artifact_input_plan_rejects_corrupt_exact_producer_kind(): ), ) ), - consumer_scope=PathPlannerGroupScope.ungrouped(), + consumer_scope=UngroupedScope(), sid=2, step_name="measure", source_bindings=EMPTY_SOURCE_BINDINGS, @@ -4727,7 +4700,7 @@ def test_same_name_source_and_output_use_exact_typed_graph_identity(): inputs = planner.artifacts.process_artifact_inputs( declarations, sid=2, - consumer_scope=PathPlannerGroupScope.ungrouped(), + consumer_scope=UngroupedScope(), source_bindings=source_bindings, variable_components=ComponentSet(), step_name="IdentifyPrimaryObjects", @@ -4758,7 +4731,7 @@ def test_same_name_source_and_output_use_exact_typed_graph_identity(): ) ), sid=3, - consumer_scope=PathPlannerGroupScope.ungrouped(), + consumer_scope=UngroupedScope(), source_bindings=EMPTY_SOURCE_BINDINGS, variable_components=ComponentSet(), step_name="ConsumePH3Objects", @@ -4781,8 +4754,7 @@ def test_artifact_input_plan_preserves_single_grouped_producer_scope(): name="illumination", path="/memory/illumination.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/illumination_channel_1.pkl"}, producer_step_index=1, producer_step_name="calculate_illumination", @@ -4798,7 +4770,7 @@ def test_artifact_input_plan_preserves_single_grouped_producer_scope(): ), ) ), - consumer_scope=PathPlannerGroupScope.from_raw( + consumer_scope=ComponentGroupScope.of( ("2", "3"), component=Microscopy.Site, ), @@ -4820,44 +4792,44 @@ def test_artifact_input_plan_preserves_single_grouped_producer_scope(): @pytest.mark.parametrize( ("available", "required", "contains"), ( - (PathPlannerGroupScope.ungrouped(), PathPlannerGroupScope.ungrouped(), True), + (UngroupedScope(), UngroupedScope(), True), ( - PathPlannerGroupScope.dynamic(Microscopy.Channel), - PathPlannerGroupScope.dynamic(Microscopy.Channel), + DynamicGroupScope(Microscopy.Channel), + DynamicGroupScope(Microscopy.Channel), True, ), ( - PathPlannerGroupScope.dynamic(Microscopy.Channel), - PathPlannerGroupScope.from_raw(("1",), component=Microscopy.Channel), + DynamicGroupScope(Microscopy.Channel), + ComponentGroupScope.of(("1",), component=Microscopy.Channel), True, ), ( - PathPlannerGroupScope.from_raw(("1", "2"), component=Microscopy.Channel), - PathPlannerGroupScope.from_raw(("2",), component=Microscopy.Channel), + ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), + ComponentGroupScope.of(("2",), component=Microscopy.Channel), True, ), ( - PathPlannerGroupScope.from_raw(("1",), component=Microscopy.Channel), - PathPlannerGroupScope.dynamic(Microscopy.Channel), + ComponentGroupScope.of(("1",), component=Microscopy.Channel), + DynamicGroupScope(Microscopy.Channel), False, ), ( - PathPlannerGroupScope.dynamic(Microscopy.Channel), - PathPlannerGroupScope.dynamic(Microscopy.Site), + DynamicGroupScope(Microscopy.Channel), + DynamicGroupScope(Microscopy.Site), False, ), ), ) def test_component_group_scope_contains_exact_required_scope( - available: PathPlannerGroupScope, - required: PathPlannerGroupScope, + available: ComponentGroupScope, + required: ComponentGroupScope, contains: bool, ) -> None: assert available.contains_scope(required) is contains def test_component_group_scope_selects_runtime_key_from_static_domain(): - scope = ComponentGroupScope.from_raw( + scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -4882,8 +4854,7 @@ def consume(main_image, image): name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/image_channel_1.pkl", "2": "/memory/image_channel_2.pkl", @@ -4913,7 +4884,7 @@ def consume(main_image, image): snapshot, 3, declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), ) with pytest.raises(ValueError, match="no exact relation-owned selection"): @@ -4943,8 +4914,7 @@ def consume(main_image, image): name="image", path="/memory/image_channel_2.pkl", artifact_type=ImageArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/image_channel_2.pkl"}, producer_step_index=1, producer_step_name="producer", @@ -4971,10 +4941,10 @@ def consume(main_image, image): snapshot, 3, declarations, - PathPlannerGroupScope.ungrouped(), + UngroupedScope(), ) - producer_scope = maps.inputs[input_spec.ref()].producer_group_scope() + producer_scope = maps.inputs[input_spec.ref()].group_scope assert producer_scope.keys == ("2",) assert producer_scope.component is Microscopy.Channel compiled = planner.artifacts.build_step_compiled_function_pattern( @@ -5008,8 +4978,7 @@ def consume(main_image, relationships): name="relationships", path="/memory/relationships_channel_2.pkl", artifact_type=RelationshipsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), paths_by_group={"2": "/memory/relationships_channel_2.pkl"}, ) compiled = compile_function_pattern( @@ -5021,19 +4990,19 @@ def consume(main_image, relationships): compiled, artifact_inputs={input_plan.ref(): input_plan}, relation_source_scopes={ - input_spec.ref(): input_plan.producer_group_scope(), - owner_spec.ref(): PathPlannerGroupScope.from_raw( + input_spec.ref(): input_plan.group_scope, + owner_spec.ref(): ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), }, - execution_group_scope=PathPlannerGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet((Microscopy.Site,)), ) edge = next(compiled.iter_invocations()).artifact_input_edges[0] assert edge.projection.invocation_scope.is_ungrouped assert edge.projection.producer_selection_scope == ( - input_plan.producer_group_scope() + input_plan.group_scope ) @@ -5049,7 +5018,7 @@ def test_realized_source_scopes_compile_cross_group_artifact_consumption(): {"source_alias": "Blue", "channel": "1"}, {"source_alias": "Green", "channel": "2"}, ) - broad_channel_scope = PathPlannerGroupScope.from_raw( + broad_channel_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -5100,7 +5069,7 @@ def produce(image, source_value): source_bindings=source_bindings, ) - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( (expected_group,), component=Microscopy.Channel, ) @@ -5156,7 +5125,7 @@ def consume(image, Nuclei, PH3): consumer_variable_components=ComponentSet((Microscopy.Site,)), ) - assert maps.group_scope == PathPlannerGroupScope.from_raw( + assert maps.group_scope == ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -5185,8 +5154,7 @@ def consume(main_image, image, site_source): name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/image_channel_1.pkl", "2": "/memory/image_channel_2.pkl", @@ -5196,8 +5164,7 @@ def consume(main_image, image, site_source): name="site_source", path="/memory/site_source.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), paths_by_group={"1": "/memory/site_source_1.pkl"}, ) input_plans = { @@ -5216,10 +5183,10 @@ def consume(main_image, image, site_source): compiled, artifact_inputs=input_plans, relation_source_scopes={ - image_spec.ref(): image_plan.producer_group_scope(), - source_spec.ref(): source_plan.producer_group_scope(), + image_spec.ref(): image_plan.group_scope, + source_spec.ref(): source_plan.group_scope, }, - execution_group_scope=PathPlannerGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), consumer_variable_components=ComponentSet((Microscopy.Site,)), ) @@ -5230,8 +5197,7 @@ def test_artifact_plan_rejects_group_component_as_variable_axis(): name="objects", path="/memory/objects.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=(None,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), variable_components=(Microscopy.Site,), ) @@ -5244,8 +5210,7 @@ def test_artifact_input_plan_preserves_multi_grouped_producer_across_components( name="illumination", path="/memory/illumination_channel_1.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/illumination_channel_1.pkl", "2": "/memory/illumination_channel_2.pkl", @@ -5264,7 +5229,7 @@ def test_artifact_input_plan_preserves_multi_grouped_producer_across_components( ), ) ), - consumer_scope=PathPlannerGroupScope.from_raw( + consumer_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ), @@ -5302,8 +5267,7 @@ def apply(image, illumination): name="illumination", path="/memory/illumination_channel_1.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), variable_components=(Microscopy.Site,), paths_by_group={ "1": "/memory/illumination_channel_1.pkl", @@ -5328,25 +5292,25 @@ def apply(image, illumination): compiled, artifact_inputs={input_plan.ref(): input_plan}, relation_source_scopes={ - illumination_spec.ref(): input_plan.producer_group_scope(), - source_spec.ref(): PathPlannerGroupScope.dynamic(Microscopy.Site), + illumination_spec.ref(): input_plan.group_scope, + source_spec.ref(): DynamicGroupScope(Microscopy.Site), }, - execution_group_scope=PathPlannerGroupScope.dynamic(Microscopy.Site), + execution_group_scope=DynamicGroupScope(Microscopy.Site), consumer_variable_components=ComponentSet((Microscopy.Channel,)), source_bindings=source_bindings, available_artifacts=ArtifactSpecCollection((source_spec, illumination_spec)), ) edge = next(compiled.iter_invocations()).artifact_input_edges[0] - assert edge.storage_plan.producer_group_scope() == input_plan.producer_group_scope() + assert edge.storage_plan.group_scope == input_plan.group_scope assert edge.projection.producer_selection_scope == ( - edge.storage_plan.producer_group_scope() + edge.storage_plan.group_scope ) - assert edge.projection.invocation_scope == ComponentGroupScope.dynamic( + assert edge.projection.invocation_scope == DynamicGroupScope( Microscopy.Site ) assert edge.projection.component_scope(Microscopy.Site) == ( - ComponentGroupScope.dynamic(Microscopy.Site) + DynamicGroupScope(Microscopy.Site) ) @@ -5367,17 +5331,15 @@ def apply_second(image): name="IllumStain1", path="/memory/IllumStain1.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), ) second = ArtifactInputPlan( name="IllumStain2", path="/memory/IllumStain2.pkl", artifact_type=ImageArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), ) - execution_scope = PathPlannerGroupScope.from_raw( + execution_scope = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) @@ -5391,8 +5353,8 @@ def apply_second(image): compiled, artifact_inputs=storage, relation_source_scopes={ - first_spec.ref(): first.producer_group_scope(), - second_spec.ref(): second.producer_group_scope(), + first_spec.ref(): first.group_scope, + second_spec.ref(): second.group_scope, }, execution_group_scope=execution_scope, consumer_variable_components=ComponentSet((Microscopy.Site,)), @@ -5425,13 +5387,13 @@ def apply_second(image): assert first_edge.projection is not None assert second_edge.projection is not None assert first_edge.projection.invocation_scope == ( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) ) assert second_edge.projection.invocation_scope == ( - ComponentGroupScope.from_raw( + ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -5450,14 +5412,13 @@ def consume(image, Objects1): name=input_spec.name, path="/memory/Objects1.pkl", artifact_type=input_spec.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), ) - invocation_scope = PathPlannerGroupScope.from_raw( + invocation_scope = ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) - producer_domain = PathPlannerGroupScope.from_raw( + producer_domain = ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ) @@ -5501,11 +5462,11 @@ def consume(image, Nuclei): {}, ).iter_invocations() ) - producer_channel = PathPlannerGroupScope.from_raw( + producer_channel = ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ) - consumer_channel = PathPlannerGroupScope.from_raw( + consumer_channel = ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -5542,11 +5503,11 @@ def consume(image, Objects): {}, ).iter_invocations() ) - producer_domain = PathPlannerGroupScope.from_raw( + producer_domain = ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ) - selected_channel = PathPlannerGroupScope.from_raw( + selected_channel = ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -5580,8 +5541,7 @@ def relate_objects(image, Cells): name=consumed.name, path="/memory/Cells.pkl", artifact_type=consumed.artifact_type, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), source_step_id=4, ) compiled = compile_function_pattern( @@ -5602,8 +5562,8 @@ def relate_objects(image, Cells): compiled = _artifact_planner_stub().artifacts.compile_invocation_input_edges( compiled, artifact_inputs={input_plan.ref(): input_plan}, - relation_source_scopes={source.ref(): input_plan.producer_group_scope()}, - execution_group_scope=PathPlannerGroupScope.from_raw( + relation_source_scopes={source.ref(): input_plan.group_scope}, + execution_group_scope=ComponentGroupScope.of( ("5",), component=Microscopy.Channel, ), @@ -5614,7 +5574,7 @@ def relate_objects(image, Cells): edge = next(compiled.iter_invocations()).artifact_input_edges[0] assert edge.projection.component_scope(Microscopy.Channel) == ( - ComponentGroupScope.from_raw(("3",), component=Microscopy.Channel) + ComponentGroupScope.of(("3",), component=Microscopy.Channel) ) @@ -5630,10 +5590,9 @@ def assemble(image, positions): name="positions", path="/memory/positions.pkl", artifact_type=SpecialArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), ) - execution_scope = PathPlannerGroupScope.dynamic(Microscopy.Channel) + execution_scope = DynamicGroupScope(Microscopy.Channel) compiled = compile_function_pattern( admitted_graph(assemble), {plan.ref(): plan for plan in (input_plan,)}, @@ -5644,14 +5603,14 @@ def assemble(image, positions): compiled, artifact_inputs={input_plan.ref(): input_plan}, relation_source_scopes={ - input_spec.ref(): input_plan.producer_group_scope(), + input_spec.ref(): input_plan.group_scope, }, execution_group_scope=execution_scope, consumer_variable_components=ComponentSet((Microscopy.Site,)), ) edge = next(compiled.iter_invocations()).artifact_input_edges[0] - expected_scope = ComponentGroupScope.dynamic(Microscopy.Channel) + expected_scope = DynamicGroupScope(Microscopy.Channel) assert edge.projection.invocation_scope == expected_scope assert edge.projection.producer_selection_scope == expected_scope @@ -5673,8 +5632,7 @@ def crop_red(image): name=mask_name, path="/memory/crop_mask.pkl", artifact_type=ImageArtifactType, - group_keys=("2", "3"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2", "3"), component=Microscopy.Channel), paths_by_group={ "2": "/memory/crop_mask_2.pkl", "3": "/memory/crop_mask_3.pkl", @@ -5689,9 +5647,9 @@ def crop_red(image): compiled, artifact_inputs={input_plan.ref(): input_plan}, relation_source_scopes={ - mask_spec.ref(): input_plan.producer_group_scope(), + mask_spec.ref(): input_plan.group_scope, }, - execution_group_scope=PathPlannerGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("2", "3"), component=Microscopy.Channel, ), diff --git a/tests/unit/test_payload_axis_compile_gate.py b/tests/unit/test_payload_axis_compile_gate.py index b6a369202f..a3802050af 100644 --- a/tests/unit/test_payload_axis_compile_gate.py +++ b/tests/unit/test_payload_axis_compile_gate.py @@ -278,7 +278,7 @@ def identity(image): source_binding_plan=CompiledSourceBindingPlan( bindings=(grayscale_binding, rgb_binding), ), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), @@ -641,7 +641,7 @@ def test_unknown_routed_group_cannot_fall_back_to_all_primary_sources( source_binding_plan=CompiledSourceBindingPlan( bindings=(grayscale_binding, rgb_binding), ), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Channel, ), diff --git a/tests/unit/test_runtime_artifact_queries.py b/tests/unit/test_runtime_artifact_queries.py index 71a7e522e5..8c5122be3e 100644 --- a/tests/unit/test_runtime_artifact_queries.py +++ b/tests/unit/test_runtime_artifact_queries.py @@ -72,6 +72,9 @@ ) from openhcs.core.runtime_artifact_values import RuntimeValue from openhcs.domains.microscopy.axes import Microscopy +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) AXIS_ID = "A01" @@ -1746,8 +1749,7 @@ def _record_native( name=native_value.name, path=f"/memory/{native_value.name}.pkl", artifact_type=kind, - group_keys=(group_key,) if group_key is not None else (), - group_component=(Microscopy.Channel if group_key is not None else None), + group_scope=ComponentGroupScope.of((group_key,) if group_key is not None else (), component=Microscopy.Channel if group_key is not None else None), ), native_value, axis_id=AXIS_ID, diff --git a/tests/unit/test_runtime_output_matching.py b/tests/unit/test_runtime_output_matching.py index a62db8ed50..cf2ffa079b 100644 --- a/tests/unit/test_runtime_output_matching.py +++ b/tests/unit/test_runtime_output_matching.py @@ -31,6 +31,9 @@ ) from openhcs.domains.microscopy.axes import Microscopy from openhcs.core.runtime_image_values import PlainImagePayload +from openhcs.core.component_group_scope import ( + ComponentGroupScope, +) def _contract( @@ -269,8 +272,7 @@ def test_compiled_output_selection_accepts_exact_runtime_group_projection( name=output.name, path="/memory/output", artifact_type=output.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/output/1", "2": "/memory/output/2", @@ -294,8 +296,7 @@ def test_compiled_output_selection_rejects_same_ref_non_owner_projection() -> No name=output.name, path="/memory/output", artifact_type=output.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group={ "1": "/memory/output/1", "2": "/memory/output/2", @@ -310,8 +311,7 @@ def test_compiled_output_selection_rejects_same_ref_non_owner_projection() -> No name=output.name, path="/memory/wrong", artifact_type=output.artifact_type, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/wrong"}, ) diff --git a/tests/unit/test_runtime_value_store.py b/tests/unit/test_runtime_value_store.py index 478020da97..4b95773301 100644 --- a/tests/unit/test_runtime_value_store.py +++ b/tests/unit/test_runtime_value_store.py @@ -33,7 +33,11 @@ RuntimeValueStore, StoredRuntimeValue, ) -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + DynamicGroupScope, + UngroupedScope, +) from openhcs.core.component_group_scope import RuntimeExecutionAxisScope from openhcs.core.component_set import ComponentSet from openhcs.core.callable_contract import CallableContract @@ -122,8 +126,7 @@ def _runtime_value(name="measurements", path="/memory/measurements.pkl"): name=name, path=path, artifact_type=MeasurementsArtifactType, - group_keys=("DAPI",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Channel), ), MeasurementTable( name=name, @@ -242,8 +245,7 @@ def test_output_plan_uses_its_single_scope_for_ungrouped_invocation(): name="Nuclei", path="/memory/Nuclei.pkl", artifact_type=ObjectLabelsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": "/memory/Nuclei_1.pkl"}, ) @@ -259,8 +261,7 @@ def test_ungrouped_output_plan_ignores_incidental_invocation_group(): name="RGBImage", path="/memory/RGBImage.pkl", artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=None, + group_scope=ComponentGroupScope.of((None,), component=None), paths_by_group={None: "/memory/RGBImage.pkl"}, ) @@ -339,8 +340,7 @@ def test_dynamic_output_plan_requires_invocation_group(): name="ChannelImage", path="/memory/ChannelImage.pkl", artifact_type=ImageArtifactType, - group_keys=(None,), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={None: "/memory/ChannelImage.pkl"}, ) @@ -379,8 +379,7 @@ def test_runtime_value_store_keeps_fixed_component_artifacts_distinct() -> None: name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), ).for_group("2") store = RuntimeValueStore() records = [] @@ -428,15 +427,14 @@ def test_runtime_value_store_keeps_fixed_component_artifacts_distinct() -> None: name="measurements", path=output_plan.path, artifact_type=MeasurementsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( input_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Channel), - producer_selection_scope=input_plan.producer_group_scope(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Channel),), + invocation_scope=DynamicGroupScope(Microscopy.Channel), + producer_selection_scope=input_plan.group_scope, + component_scopes=(DynamicGroupScope(Microscopy.Channel),), consumer_variable_components=(), ), axis_scope=RuntimeExecutionAxisScope.from_raw( @@ -456,8 +454,7 @@ def test_runtime_value_empty_fixed_scope_preserves_ordinary_key_identity() -> No name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("2",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("2",), component=Microscopy.Channel), ).for_group("2") table = MeasurementTable( name="measurements", @@ -523,7 +520,7 @@ def test_runtime_value_store_find_matching_cache_invalidates_after_replace(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), ), "memory", ), @@ -548,8 +545,7 @@ def test_runtime_artifact_query_from_input_plan_uses_group_path(): name="DNA", path="/memory/DNA.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group={"1": "/memory/DNA_s1.pkl", "2": "/memory/DNA_s2.pkl"}, ), axis_id="A01", @@ -568,7 +564,7 @@ def test_runtime_artifact_query_from_dynamic_input_plan_matches_discovered_group name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={None: "/memory/measurements.pkl"}, ), axis_id="A01", @@ -598,8 +594,7 @@ def test_observed_input_selection_preserves_groups_axis_backend_and_locations( name="RGB", path="/memory/RGB.pkl", artifact_type=ImageArtifactType, - group_keys=(None,) if dynamic else ("1", "2", "3"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,) if dynamic else ("1", "2", "3"), component=Microscopy.Site), paths_by_group=( {None: "/memory/RGB.pkl"} if dynamic @@ -607,13 +602,13 @@ def test_observed_input_selection_preserves_groups_axis_backend_and_locations( ), ) selection = ( - ComponentGroupScope.dynamic(Microscopy.Site) + DynamicGroupScope(Microscopy.Site) if dynamic - else ComponentGroupScope.from_raw(("2", "1"), component=Microscopy.Site) + else ComponentGroupScope.of(("2", "1"), component=Microscopy.Site) ) edge = _runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.ungrouped(), + invocation_scope=UngroupedScope(), producer_selection_scope=selection, component_scopes=(selection,), consumer_variable_components=(Microscopy.Site,), @@ -701,7 +696,7 @@ def test_runtime_artifact_query_from_dynamic_output_plan_uses_runtime_group_path name="RGBImage", path="/memory/A01_RGBImage.pkl", artifact_type=ImageArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), paths_by_group={None: "/memory/A01_RGBImage.pkl"}, ), axis_id="A01", @@ -724,8 +719,7 @@ def test_runtime_artifact_input_projection_selects_same_component_group(): name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=paths, ) for group_key, value in (("1", 1.0), ("2", 2.0)): @@ -744,16 +738,15 @@ def test_runtime_artifact_input_projection_selects_same_component_group(): name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), paths_by_group=paths, ) records = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Site), - producer_selection_scope=ComponentGroupScope.dynamic(Microscopy.Site), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + invocation_scope=DynamicGroupScope(Microscopy.Site), + producer_selection_scope=DynamicGroupScope(Microscopy.Site), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Channel,), ), axis_scope=RuntimeExecutionAxisScope.from_raw( @@ -777,8 +770,7 @@ def test_runtime_artifact_input_collects_exact_complete_producer_scope(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group=paths, ) for group_key in ("1", "2"): @@ -804,18 +796,17 @@ def test_runtime_artifact_input_collects_exact_complete_producer_scope(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), paths_by_group=paths, ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope( + invocation_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), - producer_selection_scope=storage_plan.producer_group_scope(), + producer_selection_scope=storage_plan.group_scope, component_scopes=(), consumer_variable_components=(Microscopy.Site,), ), @@ -842,7 +833,7 @@ def test_runtime_artifact_input_projection_collects_variable_component_groups(): name="RGBImage", path="/memory/RGBImage.pkl", artifact_type=ImageArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), paths_by_group={None: "/memory/RGBImage.pkl"}, ) for group_key, value in (("1", 1.0), ("2", 2.0)): @@ -861,15 +852,15 @@ def test_runtime_artifact_input_projection_collects_variable_component_groups(): name="RGBImage", path="/memory/RGBImage.pkl", artifact_type=ImageArtifactType, - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Site), paths_by_group={None: "/memory/RGBImage.pkl"}, ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( input_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Channel), - producer_selection_scope=input_plan.producer_group_scope(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Channel),), + invocation_scope=DynamicGroupScope(Microscopy.Channel), + producer_selection_scope=input_plan.group_scope, + component_scopes=(DynamicGroupScope(Microscopy.Channel),), consumer_variable_components=(Microscopy.Site,), ), axis_scope=RuntimeExecutionAxisScope.from_raw( @@ -903,8 +894,7 @@ def test_runtime_artifact_input_projection_transposes_producer_stack_axis() -> N name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2", "3"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2", "3"), component=Microscopy.Channel), variable_components=(Microscopy.Site,), paths_by_group=paths, ) @@ -945,17 +935,16 @@ def test_runtime_artifact_input_projection_transposes_producer_stack_axis() -> N name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2", "3"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2", "3"), component=Microscopy.Channel), variable_components=(Microscopy.Site,), paths_by_group=paths, ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Site), - producer_selection_scope=storage_plan.producer_group_scope(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + invocation_scope=DynamicGroupScope(Microscopy.Site), + producer_selection_scope=storage_plan.group_scope, + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(Microscopy.Channel,), ), axis_scope=RuntimeExecutionAxisScope.from_raw( @@ -968,7 +957,7 @@ def test_runtime_artifact_input_projection_transposes_producer_stack_axis() -> N candidates = runtime_input.candidate_execution_scopes( store, - ComponentGroupScope.dynamic(Microscopy.Site), + DynamicGroupScope(Microscopy.Site), variable_components=ComponentSet((Microscopy.Channel,)), ) assert tuple(scope.value_text for scope in candidates) == ("1", "2", "3") @@ -1016,11 +1005,11 @@ def test_artifact_candidate_scopes_preserve_projected_site_time_correlation() -> path=path, backend="memory") edge = _runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Site), - producer_selection_scope=ComponentGroupScope.ungrouped(), + invocation_scope=DynamicGroupScope(Microscopy.Site), + producer_selection_scope=UngroupedScope(), component_scopes=( - ComponentGroupScope.dynamic(Microscopy.Site), - ComponentGroupScope.from_raw(("3",), component=Microscopy.Timepoint), + DynamicGroupScope(Microscopy.Site), + ComponentGroupScope.of(("3",), component=Microscopy.Timepoint), ), consumer_variable_components=(), ) @@ -1029,7 +1018,7 @@ def test_artifact_candidate_scopes_preserve_projected_site_time_correlation() -> axis_scope=RuntimeExecutionAxisScope.from_raw("A01", component=None, value=None), ) candidates = runtime_input.candidate_execution_scopes( - store, ComponentGroupScope.dynamic(Microscopy.Site), + store, DynamicGroupScope(Microscopy.Site), variable_components=ComponentSet(), ) (coordinates,) = tuple(candidates) @@ -1045,11 +1034,11 @@ def test_artifact_candidate_scopes_keep_actual_producer_group_as_fixed_context() path = "/memory/produced_pixels.pkl" output_plan = ArtifactOutputPlan( name="image", path=path, artifact_type=ImageArtifactType, - group_component=Microscopy.Channel, group_keys=("1",), + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), ) storage_plan = ArtifactInputPlan( name="image", path=path, artifact_type=ImageArtifactType, - group_component=Microscopy.Channel, group_keys=("1",), + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), ) payload = ImagePayloadMetadata( source_path="/source/old_filename_w9.tif", @@ -1069,16 +1058,16 @@ def test_artifact_candidate_scopes_keep_actual_producer_group_as_fixed_context() runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Site), - producer_selection_scope=storage_plan.producer_group_scope(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + invocation_scope=DynamicGroupScope(Microscopy.Site), + producer_selection_scope=storage_plan.group_scope, + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(), ), axis_scope=RuntimeExecutionAxisScope.from_raw("A01", component=None, value=None), backend="memory", ) candidates = runtime_input.candidate_execution_scopes( - store, ComponentGroupScope.dynamic(Microscopy.Site), + store, DynamicGroupScope(Microscopy.Site), variable_components=ComponentSet(), ) (coordinates,) = tuple(candidates) @@ -1105,7 +1094,7 @@ def test_artifact_candidate_scopes_keep_actual_producer_group_as_fixed_context() path=path, backend="memory", ) candidates = runtime_input.candidate_execution_scopes( - store, ComponentGroupScope.dynamic(Microscopy.Site), + store, DynamicGroupScope(Microscopy.Site), variable_components=ComponentSet(), ) assert len(candidates) == 2 @@ -1133,9 +1122,9 @@ def test_runtime_artifact_input_projection_ignores_scalar_pixel_contributors() - runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Site), - producer_selection_scope=ComponentGroupScope.ungrouped(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Site),), + invocation_scope=DynamicGroupScope(Microscopy.Site), + producer_selection_scope=UngroupedScope(), + component_scopes=(DynamicGroupScope(Microscopy.Site),), consumer_variable_components=(), ), axis_scope=RuntimeExecutionAxisScope.from_raw( @@ -1208,8 +1197,7 @@ def test_runtime_artifact_input_projection_collapses_excluded_singleton_axis() - name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), variable_components=(Microscopy.Channel,), paths_by_group=paths, ) @@ -1234,18 +1222,17 @@ def test_runtime_artifact_input_projection_collapses_excluded_singleton_axis() - name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), variable_components=(Microscopy.Channel,), paths_by_group=paths, ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Channel), - producer_selection_scope=storage_plan.producer_group_scope(), + invocation_scope=DynamicGroupScope(Microscopy.Channel), + producer_selection_scope=storage_plan.group_scope, component_scopes=( - ComponentGroupScope( + ComponentGroupScope.of( ("1",), component=Microscopy.Channel, ), @@ -1285,8 +1272,7 @@ def test_runtime_artifact_input_reconstructs_singleton_producer_group_axis() -> name="image", path=path, artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), paths_by_group={"1": path}, ) store = RuntimeValueStore() @@ -1303,16 +1289,15 @@ def test_runtime_artifact_input_reconstructs_singleton_producer_group_axis() -> name="image", path=path, artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), paths_by_group={"1": path}, ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Channel), - producer_selection_scope=storage_plan.producer_group_scope(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Channel),), + invocation_scope=DynamicGroupScope(Microscopy.Channel), + producer_selection_scope=storage_plan.group_scope, + component_scopes=(DynamicGroupScope(Microscopy.Channel),), consumer_variable_components=(Microscopy.Site,), ), axis_scope=RuntimeExecutionAxisScope.from_raw( @@ -1339,8 +1324,7 @@ def test_runtime_artifact_input_keeps_singleton_scalar_selection_unstacked() -> name="image", path=path, artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": path}, ) store = RuntimeValueStore() @@ -1357,17 +1341,16 @@ def test_runtime_artifact_input_keeps_singleton_scalar_selection_unstacked() -> name="image", path=path, artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), paths_by_group={"1": path}, ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=storage_plan.producer_group_scope(), + invocation_scope=UngroupedScope(), + producer_selection_scope=storage_plan.group_scope, component_scopes=( - ComponentGroupScope(("1",), component=Microscopy.Channel), + ComponentGroupScope.of(("1",), component=Microscopy.Channel), ), consumer_variable_components=(Microscopy.Site,), ), @@ -1390,19 +1373,18 @@ def test_runtime_artifact_input_projection_selects_compiler_owned_group_for_ungr name="image", path="/memory/image.pkl", artifact_type=ImageArtifactType, - group_keys=("1", "2"), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Channel), ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), component_scopes=( - ComponentGroupScope( + ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), @@ -1419,7 +1401,7 @@ def test_runtime_artifact_input_projection_selects_compiler_owned_group_for_ungr assert ( runtime_input.edge_plan.projection.producer_selection_scope - == ComponentGroupScope( + == ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ) @@ -1436,8 +1418,7 @@ def test_runtime_artifact_input_projection_keeps_equal_keys_component_typed(): name="image", path=path, artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=component, + group_scope=ComponentGroupScope.of(("1",), component=component), paths_by_group={"1": path}, ) store.record( @@ -1454,16 +1435,15 @@ def test_runtime_artifact_input_projection_keeps_equal_keys_component_typed(): name="image", path="/memory/site_image.pkl", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), paths_by_group={"1": "/memory/site_image.pkl"}, ) records = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.dynamic(Microscopy.Channel), - producer_selection_scope=storage_plan.producer_group_scope(), - component_scopes=(ComponentGroupScope.dynamic(Microscopy.Channel),), + invocation_scope=DynamicGroupScope(Microscopy.Channel), + producer_selection_scope=storage_plan.group_scope, + component_scopes=(DynamicGroupScope(Microscopy.Channel),), consumer_variable_components=(Microscopy.Site,), ), axis_scope=RuntimeExecutionAxisScope.from_raw( @@ -1486,8 +1466,7 @@ def test_runtime_artifact_input_projection_uses_compiled_group_for_plane_scope() name="RGBImage", path=path, artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": path}, ) payload = ImagePayloadMetadata( @@ -1502,20 +1481,19 @@ def test_runtime_artifact_input_projection_uses_compiled_group_for_plane_scope() name="RGBImage", path=path, artifact_type=ImageArtifactType, - group_keys=("3",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("3",), component=Microscopy.Channel), paths_by_group={"3": path}, ) runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=ComponentGroupScope( + invocation_scope=UngroupedScope(), + producer_selection_scope=ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ), component_scopes=( - ComponentGroupScope( + ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ), @@ -1532,7 +1510,7 @@ def test_runtime_artifact_input_projection_uses_compiled_group_for_plane_scope() assert ( runtime_input.edge_plan.projection.producer_selection_scope - == ComponentGroupScope( + == ComponentGroupScope.of( ("3",), component=Microscopy.Channel, ) @@ -1726,7 +1704,7 @@ def _ungrouped_runtime_artifact_input( axis_scope: RuntimeExecutionAxisScope = RuntimeExecutionAxisScope(axis_id="A01"), source_binding_plan: CompiledSourceBindingPlan = CompiledSourceBindingPlan(), ) -> RuntimeArtifactInput: - ungrouped = ComponentGroupScope.ungrouped() + ungrouped = UngroupedScope() return RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, @@ -1823,8 +1801,7 @@ def _grouped_label_input_with_distinct_consumer_channel(): name="Cells", path=path, artifact_type=ObjectLabelsArtifactType, - group_component=Microscopy.Channel, - group_keys=("0",), + group_scope=ComponentGroupScope.of(("0",), component=Microscopy.Channel), paths_by_group={"0": path}, source_step_id=20, ) @@ -1832,8 +1809,7 @@ def _grouped_label_input_with_distinct_consumer_channel(): name=storage_plan.name, path=path, artifact_type=storage_plan.artifact_type, - group_component=storage_plan.group_component, - group_keys=storage_plan.group_keys, + group_scope=ComponentGroupScope.of(storage_plan.group_keys, component=storage_plan.group_component), paths_by_group=storage_plan.paths_by_group, ).for_group("0") value = RuntimeValue.normalize_for_execution_scope( @@ -1854,8 +1830,8 @@ def _grouped_label_input_with_distinct_consumer_channel(): runtime_input = RuntimeArtifactInput( edge_plan=_runtime_input_edge( storage_plan, - invocation_scope=ComponentGroupScope.ungrouped(), - producer_selection_scope=storage_plan.producer_group_scope(), + invocation_scope=UngroupedScope(), + producer_selection_scope=storage_plan.group_scope, component_scopes=(), consumer_variable_components=(Microscopy.ZIndex,), ), @@ -1877,7 +1853,7 @@ def test_exact_producer_group_does_not_constrain_consumer_source_channel(): assert runtime_input.records(store) == (record,) assert runtime_input.resolve_value(store) is record.data candidates = runtime_input.candidate_execution_scopes( - store, ComponentGroupScope.ungrouped(), + store, UngroupedScope(), variable_components=ComponentSet((Microscopy.ZIndex,)), ) (scope,) = candidates @@ -1953,7 +1929,7 @@ def test_different_producer_group_axis_preserves_fixed_source_channel_constraint ) storage_plan = replace( runtime_input.edge_plan.storage_plan, - group_component=Microscopy.Site, group_keys=("1",), + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), paths_by_group={"1": record.location.path}, ) runtime_input = replace( @@ -1963,7 +1939,7 @@ def test_different_producer_group_axis_preserves_fixed_source_channel_constraint storage_plan=storage_plan, projection=replace( runtime_input.edge_plan.projection, - producer_selection_scope=storage_plan.producer_group_scope(), + producer_selection_scope=storage_plan.group_scope, ), ), ) @@ -2147,7 +2123,7 @@ def test_paired_channel_projection_rejects_a_different_producer_site_plane(): storage_plan=replace(edge.storage_plan, variable_components=(Microscopy.Site,)), projection=replace( edge.projection, - component_scopes=(ComponentGroupScope(("1",), component=Microscopy.Site),), + component_scopes=(ComponentGroupScope.of(("1",), component=Microscopy.Site),), ), ), ) @@ -2458,7 +2434,7 @@ def test_dynamic_input_query_distinguishes_compiled_paths_with_same_backend(): name="measurements", path=record.location.path, artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={None: record.location.path, "DAPI": record.location.path}, ), axis_id="A01", @@ -2480,8 +2456,7 @@ def test_dynamic_input_query_retains_discovery_order_and_ignores_wrong_component name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Site, - group_keys=("DAPI",), + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Site), ), value.data, axis_id="A01", @@ -2493,7 +2468,7 @@ def test_dynamic_input_query_retains_discovery_order_and_ignores_wrong_component name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), ), axis_id="A01", backend="memory", @@ -2507,7 +2482,7 @@ def test_dynamic_query_snapshots_address_mapping_without_mutating_source_plan(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group=paths, ) old_query = RuntimeArtifactQuery.from_input_plan( @@ -2546,7 +2521,7 @@ def test_dynamic_query_retains_absent_and_empty_path_declarations(paths): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group=paths, ) query = RuntimeArtifactQuery.from_input_plan(plan, axis_id="A01", backend="memory") @@ -2561,7 +2536,7 @@ def test_input_plan_owns_independent_immutable_runtime_address_snapshot(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group=paths, source_step_id=7, source_step_scope_id="producer-scope", @@ -2596,7 +2571,7 @@ def test_store_transport_excludes_all_derived_lookup_caches(): name="measurements", path=record.location.path, artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), paths_by_group={"DAPI": record.location.path}, ), axis_id="A01", @@ -2637,7 +2612,7 @@ def test_unified_store_cache_keeps_both_query_domains_and_empty_results(): name="measurements", path=record.location.path, artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of((None,), component=Microscopy.Channel), ), axis_id="A01", backend="memory", @@ -2673,8 +2648,7 @@ def test_unified_store_cache_eviction_recomputes_order_without_changing_record_a name="first", path=first.location.path, artifact_type=MeasurementsArtifactType, - group_component=Microscopy.Channel, - group_keys=("DAPI",), + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Channel), ), axis_id="A01", backend="memory", diff --git a/tests/unit/test_runtime_values.py b/tests/unit/test_runtime_values.py index 54adc29e5b..cc99f7efb5 100644 --- a/tests/unit/test_runtime_values.py +++ b/tests/unit/test_runtime_values.py @@ -22,7 +22,10 @@ RelationshipsArtifactType, SpatialGridArtifactType, ) -from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + RuntimeExecutionAxisScope, +) from openhcs.core.runtime_artifact_values import ( ArtifactKey, RuntimeValue, @@ -2843,8 +2846,7 @@ def test_normalize_artifact_value_builds_key_for_nominal_payload(): name="measurements", path="/memory/measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("DAPI",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("DAPI",), component=Microscopy.Channel), ) table = MeasurementTable( name="measurements", @@ -2886,8 +2888,7 @@ def test_group_scoped_artifact_consumes_singleton_payload_axis() -> None: name="Corrected", path="/memory/Corrected.tif", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), ) value = RuntimeValue.normalize( @@ -2905,8 +2906,7 @@ def test_group_scoped_artifact_rejects_multi_plane_payload_axis() -> None: name="Corrected", path="/memory/Corrected.tif", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Site), ) with pytest.raises(ValueError, match="cannot retain a declared runtime-slice"): @@ -2922,8 +2922,7 @@ def test_group_scoped_stack_artifact_preserves_declared_payload_axis() -> None: name="Corrected", path="/memory/Corrected.tif", artifact_type=ImageArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), variable_components=(Microscopy.Site,), ) @@ -2944,8 +2943,7 @@ def test_group_scoped_measurements_preserve_runtime_row_axis() -> None: name="Measurements", path="/memory/Measurements.pkl", artifact_type=MeasurementsArtifactType, - group_keys=("1",), - group_component=Microscopy.Channel, + group_scope=ComponentGroupScope.of(("1",), component=Microscopy.Channel), ) table = MeasurementTable( name="Measurements", @@ -2976,8 +2974,7 @@ def test_grouped_scalar_artifact_records_compose_one_runtime_axis() -> None: name="Corrected", path=f"/memory/Corrected_{site}.tif", artifact_type=ImageArtifactType, - group_keys=(site,), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of((site,), component=Microscopy.Site), ), ImagePayloadMetadata().payload_with( np.zeros((4, 5), dtype=np.float32), diff --git a/tests/unit/test_source_bound_matched_image_set_anchors.py b/tests/unit/test_source_bound_matched_image_set_anchors.py index 19a3bdce53..b3fbdf96c2 100644 --- a/tests/unit/test_source_bound_matched_image_set_anchors.py +++ b/tests/unit/test_source_bound_matched_image_set_anchors.py @@ -18,7 +18,10 @@ SpecialArtifactType, ) from openhcs.core.callable_contract import FunctionStepExecutionScope -from openhcs.core.component_group_scope import ComponentGroupScope +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + UngroupedScope, +) from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.pipeline.function_contracts import ( artifact_inputs, @@ -111,7 +114,7 @@ def consume(*, payload): pattern_filter = _anchor_executor( plan=SimpleNamespace( source_binding_plan=source_binding_plan, - execution_group_scope=ComponentGroupScope.ungrouped(), + execution_group_scope=UngroupedScope(), ), parser=SourceSchemaFilenameParser(), output_manifest=None, @@ -229,14 +232,11 @@ def measure_images(image): name=output_spec.name, path=f"/memory/{output_spec.name}.pkl", artifact_type=output_spec.artifact_type, - group_keys=( - tuple( + group_scope=ComponentGroupScope.of(tuple( binding.component_identity[0].value for binding in compiled_bindings ) if shared_output or output_spec.name == "AggregateMeasurements" - else (binding.component_identity[0].value,) - ), - group_component=Microscopy.Channel, + else (binding.component_identity[0].value,), component=Microscopy.Channel), relations=output_spec.relations, ) for output_spec, binding in zip( @@ -256,7 +256,7 @@ def measure_images(image): step_name="MeasureImages", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( tuple(binding.component_identity[0].value for binding in compiled_bindings), component=Microscopy.Channel, ), @@ -425,7 +425,7 @@ def measure_hoechst(image): step_name="GroupedMeasurements", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=source_binding_plan, - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("2", "1"), component=Microscopy.Channel, ), @@ -485,7 +485,7 @@ def identify_bodies(image): step_name="NeuronBodies", main_input_dependency=StepInputDependency.pipeline_start(), source_binding_plan=CompiledSourceBindingPlan(bindings=(binding,)), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("2",), component=Microscopy.Channel, ), @@ -581,8 +581,7 @@ def align_source_set(image): name=measurements.name, path="/memory/SourceSetMeasurements.pkl", artifact_type=measurements.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), relations=measurements.relations, ) plan = SimpleNamespace( @@ -594,7 +593,7 @@ def align_source_set(image): bindings=bindings, match_plan=PairByOrder(), ), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ), @@ -701,8 +700,7 @@ def align_source_set(image): name=spec.name, path=f"/memory/{spec.name}.pkl", artifact_type=spec.artifact_type, - group_keys=("1", "2"), - group_component=Microscopy.Site, + group_scope=ComponentGroupScope.of(("1", "2"), component=Microscopy.Site), relations=spec.relations, ) for spec in output_specs @@ -716,7 +714,7 @@ def align_source_set(image): bindings=bindings, match_plan=PairByOrder(), ), - execution_group_scope=ComponentGroupScope.from_raw( + execution_group_scope=ComponentGroupScope.of( ("1", "2"), component=Microscopy.Site, ), From b8d308e74c7cbc786f97d0aba337e761df26a23f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 15:31:55 -0400 Subject: [PATCH 05/13] WIP: K7 context phase types (production side; tests not yet migrated) Co-Authored-By: Claude Opus 5.5 --- openhcs/core/compiled_execution.py | 12 +- openhcs/core/context/__init__.py | 8 +- openhcs/core/context/processing_context.py | 245 +++++++++--------- .../core/dataset_sources/openhcs_format.py | 8 +- openhcs/core/function_step_transport.py | 7 +- openhcs/core/orchestrator/bundle_execution.py | 62 ++--- openhcs/core/orchestrator/execution_result.py | 9 +- openhcs/core/orchestrator/orchestrator.py | 8 +- openhcs/core/orchestrator/worker_execution.py | 100 ++++--- openhcs/core/orchestrator/worker_lanes.py | 10 +- openhcs/core/pipeline/compilation_session.py | 8 +- openhcs/core/pipeline/compiler.py | 66 ++--- openhcs/core/post_execute.py | 14 +- openhcs/core/runtime_adapters.py | 4 +- openhcs/core/runtime_batch_contracts.py | 8 +- openhcs/core/runtime_execution_validation.py | 23 +- openhcs/core/source_binding_selection.py | 6 +- openhcs/core/source_workspace_projection.py | 10 +- openhcs/core/steps/abstract.py | 10 +- .../function_artifact_materialization.py | 42 +-- openhcs/core/steps/function_execution.py | 6 +- openhcs/core/steps/function_io.py | 6 +- .../core/steps/function_output_manifest.py | 6 +- openhcs/core/steps/function_outputs.py | 52 ++-- openhcs/core/steps/function_runtime.py | 8 +- openhcs/core/steps/function_step.py | 4 +- .../core/steps/stream_component_semantics.py | 16 +- openhcs/core/streaming_config_factory.py | 6 +- openhcs/core/virtual_workspace_metadata.py | 4 +- .../microscopy/analysis_consolidation.py | 6 +- .../interop/cellprofiler/analyst_export.py | 6 +- .../cellprofiler/image_set_numbering.py | 4 +- .../cellprofiler/export_to_database.py | 6 +- .../backends/cellprofiler/save_images.py | 6 +- .../cellprofiler/spreadsheet_export.py | 8 +- openhcs/runtime/zmq_execution_observation.py | 10 +- openhcs/runtime/zmq_execution_server.py | 2 +- .../test_neurite_outgrowth_synthetic_plate.py | 2 +- .../test_cellprofiler_module_execution.py | 6 +- .../unit/test_cellprofiler_runtime_adapter.py | 10 +- tests/unit/test_debug_runtime.py | 2 +- .../unit/test_funcstep_contract_validator.py | 2 +- tests/unit/test_function_artifact_outputs.py | 4 +- .../unit/test_k7_execution_protocol_guards.py | 195 ++++++++++++++ .../unit/test_path_planner_materialization.py | 78 +++--- tests/unit/test_routed_composite_execution.py | 7 +- .../unit/test_runtime_execution_validation.py | 6 +- tests/unit/test_runtime_value_store.py | 10 +- 48 files changed, 644 insertions(+), 494 deletions(-) create mode 100644 tests/unit/test_k7_execution_protocol_guards.py diff --git a/openhcs/core/compiled_execution.py b/openhcs/core/compiled_execution.py index d9d757ae21..940c40a4bc 100644 --- a/openhcs/core/compiled_execution.py +++ b/openhcs/core/compiled_execution.py @@ -9,7 +9,7 @@ resolve_lazy_configurations_for_serialization, ) from openhcs.core.config import GlobalPipelineConfig, MultiprocessingStartMethod -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import CompiledContext from openhcs.core.steps.abstract import AbstractStep from openhcs.core.worker_start_policy import ( WorkerStartDecision, @@ -80,7 +80,7 @@ def from_global_config( cls, global_config: GlobalPipelineConfig, *, - compiled_contexts: Mapping[str, ProcessingContext] | None, + compiled_contexts: Mapping[str, CompiledContext] | None, server_mode: bool, ) -> "CompiledRuntimeEnvironmentPlan": execution_facts = WorkerStartExecutionFacts.from_compiled_contexts( @@ -128,8 +128,8 @@ class CompiledExecutionBundle: """ pipeline_definition: Sequence[AbstractStep] - runtime_contexts: Mapping[str, ProcessingContext] - transport_contexts: Mapping[str, ProcessingContext] + runtime_contexts: Mapping[str, CompiledContext] + transport_contexts: Mapping[str, CompiledContext] worker_assignments: Mapping[str, list[str]] runtime_environment: CompiledRuntimeEnvironmentPlan @@ -161,7 +161,7 @@ def from_runtime_contexts( cls, *, pipeline_definition: Sequence[AbstractStep], - runtime_contexts: Mapping[str, ProcessingContext], + runtime_contexts: Mapping[str, CompiledContext], worker_assignments: Mapping[str, list[str]], runtime_environment: CompiledRuntimeEnvironmentPlan, ) -> "CompiledExecutionBundle": @@ -185,7 +185,7 @@ def from_unassigned_runtime_contexts( cls, *, pipeline_definition: Sequence[AbstractStep], - runtime_contexts: Mapping[str, ProcessingContext], + runtime_contexts: Mapping[str, CompiledContext], runtime_environment: CompiledRuntimeEnvironmentPlan, ) -> "CompiledExecutionBundle": """Create a bundle before worker ownership has been assigned.""" diff --git a/openhcs/core/context/__init__.py b/openhcs/core/context/__init__.py index b3d9879cc2..f2165e967d 100644 --- a/openhcs/core/context/__init__.py +++ b/openhcs/core/context/__init__.py @@ -1,7 +1 @@ -""" -Core context package containing components for context management. -""" - -from .processing_context import ProcessingContext - -__all__ = ["ProcessingContext"] +"""Processing contexts, one class per phase (see processing_context).""" diff --git a/openhcs/core/context/processing_context.py b/openhcs/core/context/processing_context.py index 1a20391c5e..5223ceec00 100644 --- a/openhcs/core/context/processing_context.py +++ b/openhcs/core/context/processing_context.py @@ -1,7 +1,11 @@ -""" -Processing Context for OpenHCS. - -This module defines the ProcessingContext class, which maintains state during pipeline execution. +"""Processing contexts for OpenHCS, one class per phase. + +``ProcessingContext`` holds what the compiler decides for one multiprocessing-axis +value. ``CompilingContext`` is that state while the compiler fills it in; +``CompilingContext.freeze`` returns an immutable ``CompiledContext``, which is +what bundles and workers carry. ``CompiledContext.executing`` starts one +execution: an ``ExecutingContext`` holds the compiled facts plus the runtime +stores, caches and debug sink of that execution only. """ from __future__ import annotations @@ -62,34 +66,42 @@ def launch_message(self) -> str: class ProcessingContext: - """ - Maintains state during pipeline execution. - - The ProcessingContext is the canonical owner of all state during pipeline execution. - After compilation and freezing, it should be treated as immutable by processing steps. - - OWNERSHIP: This class may ONLY be instantiated by PipelineOrchestrator. - All other components must receive a context instance, never create one. + """What the compiler decides for one multiprocessing-axis value. Attributes: - step_plans: Dictionary mapping step indices to compiled execution plans. - axis_id: Identifier of the multiprocessing axis value being processed. - filemanager: Instance of FileManager for VFS operations. + step_plans: Compiled plan of each step, by step index. + axis_id: The multiprocessing-axis value being processed. + filemanager: FileManager for VFS operations. post_execute_hooks: Domain hooks bound for this compilation. - output_dataset_root: Planned runtime output plate root. - pipeline_sequential_mode: Flag indicating pipeline-wide vs step-wide sequential processing. - pipeline_sequential_combinations: Pre-computed sequential combinations for pipeline-wide mode. - current_sequential_combination: Active combination during pipeline-wide sequential execution. - _is_frozen: Internal flag indicating if the context is immutable. + output_dataset_root: Planned runtime output dataset root. + pipeline_sequential_mode: Whether sequential combinations are pipeline-wide. + pipeline_sequential_combinations: Combinations for pipeline-wide mode. + current_sequential_combination: The combination this context compiles. """ - _is_frozen: bool = False + step_plans: dict[int, CompiledStepPlan] + axis_id: str | None + filemanager: FileManager | None @classmethod def require_parameter_name(cls) -> str: """Return the callable ABI name for runtime context injection.""" return "context" + def require_axis_id(self) -> str: + """Return the compiled multiprocessing-axis identity.""" + + if not self.axis_id: + raise ValueError(f"{type(self).__name__} requires an axis_id.") + return self.axis_id + + +class CompilingContext(ProcessingContext): + """A processing context while the compiler fills it in. + + OWNERSHIP: only ``PipelineOrchestrator.create_context`` builds one. + """ + def __init__( self, step_plans: dict[int, CompiledStepPlan] | None = None, @@ -100,34 +112,7 @@ def __init__( output_dataset_root: str | None = None, transport_config: ZMQConfig = OPENHCS_ZMQ_CONFIG, ): - """ - Initialize the processing context. - - Args: - step_plans: Dictionary mapping step indices to compiled execution plans. - axis_id: Identifier of the multiprocessing axis value being processed. - filemanager: FileManager instance for VFS operations. - post_execute_hooks: Domain hooks bound for this compilation. - output_dataset_root: Planned runtime output plate root. - """ - self._is_frozen = False - - if step_plans is None: - self.step_plans = {} - else: - self.step_plans = dict(step_plans) - self.runtime_value_store = RuntimeValueStore() - self.runtime_image_stack_cache = RuntimeImageStackCache() - self.runtime_function_output_identity_cache = FunctionOutputIdentityCache() - self.runtime_pattern_discovery_cache = RuntimePatternDiscoveryCache() - self.runtime_source_binding_context_cache = RuntimeSourceBindingContextCache() - self._runtime_step_values: dict[type[object], object] | None = None - from openhcs.core.steps.abstract import StepExecutionObservation - - self.completed_step_outputs = StepExecutionObservation.empty() - self._runtime_source_workspace_projections: ( - WorkspaceSourceProjections | None - ) = None + self.step_plans = {} if step_plans is None else dict(step_plans) self.source_image_set_identity_policy = SourceImageSetIdentityPolicy() self.axis_id = axis_id self.filemanager = filemanager @@ -142,14 +127,94 @@ def __init__( self.post_execute_hooks = tuple(post_execute_hooks) self.output_dataset_root = output_dataset_root self.transport_config = transport_config - - self.execution_runtime: WorkerLaneExecutionContext | None = None - self.debug_event_sink = NO_OP_DEBUG_EVENT_SINK - self.pipeline_sequential_mode = False self.pipeline_sequential_combinations = None self.current_sequential_combination = None + def inject_plan(self, step_id: int, plan: CompiledStepPlan) -> None: + """Add one step's compiled plan.""" + self.step_plans[step_id] = plan + + def freeze(self) -> "CompiledContext": + """Return the immutable compiled context once every fact is set.""" + + if not self.axis_id: + raise RuntimeError("Cannot freeze ProcessingContext: 'axis_id' is not set.") + if self.filemanager is None: + raise RuntimeError( + "Cannot freeze ProcessingContext: 'filemanager' is not set." + ) + return CompiledContext(vars(self)) + + +class CompiledContext(ProcessingContext): + """The compiler's immutable result for one axis value (or combination).""" + + def __init__(self, compiled_facts: dict[str, object]): + object.__setattr__(self, "_compiled_facts", frozenset(compiled_facts)) + self.__dict__.update(compiled_facts) + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError( + f"{type(self).__name__} is immutable; cannot set {name!r}." + ) + + def compiled_facts(self) -> dict[str, object]: + return {name: self.__dict__[name] for name in self._compiled_facts} + + def with_step_plans( + self, step_plans: dict[int, CompiledStepPlan] + ) -> "CompiledContext": + """Return this compiled context with its step plans replaced.""" + return CompiledContext(self.compiled_facts() | {"step_plans": step_plans}) + + def executing( + self, + execution_runtime: "WorkerLaneExecutionContext | None" = None, + ) -> "ExecutingContext": + """Start one execution of this compiled context.""" + return ExecutingContext(self, execution_runtime) + + +class ExecutingContext(CompiledContext): + """One execution of a compiled context, with that execution's state. + + ``execution_runtime`` is the worker lane running it; dataset-scoped steps + run in the parent without a lane. + """ + + def __init__( + self, + compiled: CompiledContext, + execution_runtime: "WorkerLaneExecutionContext | None" = None, + ): + from openhcs.core.steps.abstract import StepExecutionObservation + + CompiledContext.__init__(self, compiled.compiled_facts()) + self.runtime_value_store = RuntimeValueStore() + self.runtime_image_stack_cache = RuntimeImageStackCache() + self.runtime_function_output_identity_cache = FunctionOutputIdentityCache() + self.runtime_pattern_discovery_cache = RuntimePatternDiscoveryCache() + self.runtime_source_binding_context_cache = RuntimeSourceBindingContextCache() + self.completed_step_outputs = StepExecutionObservation.empty() + self.debug_event_sink = NO_OP_DEBUG_EVENT_SINK + self.execution_runtime = execution_runtime + self._runtime_step_values: dict[type[object], object] | None = None + self._runtime_source_workspace_projections: ( + WorkspaceSourceProjections | None + ) = None + if execution_runtime is not None: + execution_runtime.install_debug_sink(self) + + def __setattr__(self, name: str, value: object) -> None: + if name in self._compiled_facts: + CompiledContext.__setattr__(self, name, value) + object.__setattr__(self, name, value) + + def compiled(self) -> CompiledContext: + """Return the compiled facts this execution runs.""" + return CompiledContext(self.compiled_facts()) + @property def runtime_source_workspace_projections( self, @@ -163,21 +228,6 @@ def runtime_source_workspace_projections( self._runtime_source_workspace_projections = authority return authority - def bind_execution_runtime( - self, - runtime: WorkerLaneExecutionContext, - ) -> None: - """Bind worker-owned execution identity after compilation freeze.""" - - self.execution_runtime = runtime - self.reset_completed_step_outputs() - - def reset_completed_step_outputs(self) -> None: - """Start an execution's saved-output facts without retaining old payloads.""" - from openhcs.core.steps.abstract import StepExecutionObservation - - self.completed_step_outputs = StepExecutionObservation.empty() - def record_completed_step_outputs( self, observation: "StepExecutionObservation" ) -> None: @@ -263,62 +313,5 @@ def install_debug_event_sink(self, debug_event_sink: DebugEventSink) -> None: ) self.debug_event_sink = debug_event_sink - def inject_plan(self, step_id: int, plan: CompiledStepPlan) -> None: - """ - Inject a step plan into the context. - - This method is the canonical way to add step plans to the context during compilation. - All step configuration must be injected into the context using this method. - - Args: - step_id: The step index used as the compiled-plan key. - plan: The step execution plan - - Raises: - AttributeError: If the context is frozen. - """ - if self._is_frozen: - raise AttributeError("Cannot inject plan into a frozen ProcessingContext.") - self.step_plans[step_id] = plan - - def freeze(self) -> None: - """ - Freezes the context, making its attributes immutable. - - This should be called after all compilation and plan injection is complete. - Essential attributes like step_plans, filemanager, and axis_id must be set. - - Raises: - RuntimeError: If essential attributes are not set before freezing. - """ - if not self.axis_id: - raise RuntimeError("Cannot freeze ProcessingContext: 'axis_id' is not set.") - if self.filemanager is None: - raise RuntimeError( - "Cannot freeze ProcessingContext: 'filemanager' is not set." - ) - if self.step_plans is None: - raise RuntimeError( - "Cannot freeze ProcessingContext: 'step_plans' is not set." - ) - - self._is_frozen = True - - def require_axis_id(self) -> str: - """Return the compiled multiprocessing-axis identity.""" - - if not self.axis_id: - raise ValueError("Compiled ProcessingContext requires an axis_id.") - return self.axis_id - - def is_frozen(self) -> bool: - """ - Check if the context is frozen. - - Returns: - True if the context is frozen, False otherwise. - """ - return self._is_frozen - -DebugExecutionContext.register(ProcessingContext) +DebugExecutionContext.register(ExecutingContext) diff --git a/openhcs/core/dataset_sources/openhcs_format.py b/openhcs/core/dataset_sources/openhcs_format.py index f62076e8b3..d6a2c9513c 100644 --- a/openhcs/core/dataset_sources/openhcs_format.py +++ b/openhcs/core/dataset_sources/openhcs_format.py @@ -55,7 +55,7 @@ from openhcs.core.axes import Axis, AxisFamily if TYPE_CHECKING: - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext logger = logging.getLogger(__name__) @@ -1041,7 +1041,7 @@ def _optional_metadata_field( class OpenHCSMetadataGenerationRequest: """Request for writing one OpenHCS metadata subdirectory.""" - context: "ProcessingContext" + context: "ExecutingContext" output_dir: str write_backend: str is_main: bool @@ -1074,7 +1074,7 @@ def __init__(self, filemanager: FileManager): def create_metadata( self, - context: "ProcessingContext", + context: "ExecutingContext", output_dir: str, write_backend: str, is_main: bool = False, @@ -1155,7 +1155,7 @@ def _extract_metadata_from_disk_state( if context.metadata_cache is None: raise RuntimeError( - "ProcessingContext metadata_cache must be populated by create_context()" + "ExecutingContext metadata_cache must be populated by create_context()" ) actual_files = self.filemanager.list_image_files( diff --git a/openhcs/core/function_step_transport.py b/openhcs/core/function_step_transport.py index a3697871cf..66a732d89f 100644 --- a/openhcs/core/function_step_transport.py +++ b/openhcs/core/function_step_transport.py @@ -2,7 +2,6 @@ from __future__ import annotations -from copy import copy from dataclasses import replace from collections.abc import Mapping, Sequence from typing import Any @@ -99,8 +98,7 @@ def normalize_context( """Derive transport plans without replacing prepared runtime contracts.""" if contracts is None: contracts = {} - transport_context = copy(context) - transport_context.step_plans = { + return context.with_step_plans({ step_id: replace( step_plan, compiled_function_pattern=cls.normalize_compiled_pattern( @@ -108,8 +106,7 @@ def normalize_context( ), ) for step_id, step_plan in context.step_plans.items() - } - return transport_context + }) @classmethod def normalize_step(cls, step: Any) -> Any: diff --git a/openhcs/core/orchestrator/bundle_execution.py b/openhcs/core/orchestrator/bundle_execution.py index 9c43e0976f..d6eb4e0c47 100644 --- a/openhcs/core/orchestrator/bundle_execution.py +++ b/openhcs/core/orchestrator/bundle_execution.py @@ -31,7 +31,8 @@ ) from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.context.processing_context import ( - ProcessingContext, + CompiledContext, + ExecutingContext, RequiredVisualizer, ) from openhcs.core.debug import DebugExecutionPolicy @@ -125,18 +126,24 @@ class BundleExecutionExtras: class BundleExecutionResults(dict[str, ExecutionResult]): - """Axis results plus parent-owned diagnostics from one compiled execution.""" + """Axis results plus parent-owned diagnostics from one compiled execution. + + ``executed_contexts`` are the parent's execution contexts, holding the + runtime records merged back from the workers. + """ def __init__( self, results: Mapping[str, ExecutionResult] | None = None, *, extras: BundleExecutionExtras | None = None, + executed_contexts: Mapping[str, ExecutingContext] | None = None, ) -> None: super().__init__(results or {}) self.extras = extras or BundleExecutionExtras( viewer_states_by_port=MappingProxyType({}) ) + self.executed_contexts = MappingProxyType(dict(executed_contexts or {})) def is_success(self) -> bool: """Whether every returned axis completed successfully.""" @@ -178,7 +185,7 @@ def pipeline_definition(self) -> List[AbstractStep]: return list(self.execution_bundle.pipeline_definition) @property - def compiled_contexts(self) -> Dict[str, ProcessingContext]: + def compiled_contexts(self) -> Dict[str, CompiledContext]: return dict(self.execution_bundle.runtime_contexts) @@ -187,7 +194,7 @@ class ValidatedBundleExecution(ProgressExecutionContext): """Validated execution inputs plus defaults derived from orchestrator state.""" pipeline_definition: List[AbstractStep] - compiled_contexts: Dict[str, ProcessingContext] + compiled_contexts: Dict[str, CompiledContext] actual_max_workers: int progress_queue: ProgressQueue runtime_environment: CompiledRuntimeEnvironmentPlan @@ -232,8 +239,10 @@ def execute_bundle_request( raise set_progress_queue(validated.progress_queue) - for context in validated.compiled_contexts.values(): - context.reset_completed_step_outputs() + executing_contexts = { + context_key: context.executing() + for context_key, context in validated.compiled_contexts.items() + } try: orchestrator._state = OrchestratorState.EXECUTING logger.info( @@ -272,7 +281,7 @@ def execute_bundle_request( lane_results = executor_resources.run_worker_lanes( pipeline_definition=validated.pipeline_definition, worker_lane_execution_plan=worker_lane_execution_plan, - parent_contexts=validated.compiled_contexts, + parent_contexts=executing_contexts, ) # Lanes finish in any order; results follow the compiled axis order. execution_results = BundleExecutionResults( @@ -292,7 +301,7 @@ def execute_bundle_request( cancellation.raise_if_requested("after worker execution") if execution_results.is_success(): dataset_runtime_observation = execute_dataset_scoped_steps( - validated.compiled_contexts, + executing_contexts, progress_queue=validated.progress_queue, progress_context=validated, executor_resources=( @@ -318,16 +327,16 @@ def execute_bundle_request( if execution_results.is_success(): PostExecuteHook.run_all( - validated.compiled_contexts, + executing_contexts, execution_results, dataset_runtime_observation, ) finally: OpenHCSMetadataTarget.finalize_completed_dataset( - validated.compiled_contexts, + executing_contexts, runtime_observations=( RuntimeExecutionObservation.from_completed_outputs( - validated.compiled_contexts + executing_contexts ), ), ) @@ -357,6 +366,7 @@ def execute_bundle_request( viewer_states_by_port=viewer_states_by_port, runtime_observation=dataset_runtime_observation, ), + executed_contexts=executing_contexts, ) except ExecutionCancelledError: orchestrator._state = OrchestratorState.READY @@ -417,7 +427,7 @@ def validate_bundle_execution( def validate_dataset_scoped_contexts( - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, CompiledContext], ) -> tuple[int, ...]: """Validate dataset-scoped lifecycle invariants shared by compilation contexts.""" @@ -496,7 +506,7 @@ def validate_dataset_scoped_contexts( def execute_dataset_scoped_steps( - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, ExecutingContext], *, progress_queue: ProgressQueue, progress_context: ProgressExecutionContext, @@ -814,7 +824,7 @@ def _validate_dataset_invocation( def _runtime_record_snapshot( - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, ExecutingContext], ) -> Mapping[str, tuple[StoredRuntimeValue, ...]]: records: OrderedDict[ str, @@ -836,9 +846,9 @@ def _runtime_record_snapshot( def _dataset_output_owner( - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, ExecutingContext], step_index: int, -) -> tuple[ProcessingContext, CompiledStepPlan]: +) -> tuple[ExecutingContext, CompiledStepPlan]: owners = tuple( (context, context.step_plans[step_index]) for context in compiled_contexts.values() @@ -854,7 +864,7 @@ def _dataset_output_owner( def _dataset_artifact_batch( *, - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, ExecutingContext], step_index: int, invocation_position: int, contract: CallableContract, @@ -951,7 +961,7 @@ def actual_max_workers( def bootstrap_execution_visualizers( *, orchestrator: "PipelineOrchestrator", - compiled_contexts: Dict[str, ProcessingContext], + compiled_contexts: Dict[str, CompiledContext], visualizer: ExecutionVisualizerABC | None, progress_queue: ProgressQueue, progress_context: ProgressExecutionContext, @@ -988,31 +998,25 @@ def bootstrap_execution_visualizers( def iter_required_visualizers( *, orchestrator: "PipelineOrchestrator", - compiled_contexts: Dict[str, ProcessingContext], + compiled_contexts: Dict[str, CompiledContext], progress_queue: ProgressQueue, progress_context: ProgressExecutionContext, ) -> Iterator[ExecutionVisualizerABC]: """Create one viewer for each distinct visualizer requirement.""" - unique_configs: dict[tuple[str, int], tuple[RequiredVisualizer, object]] = {} + unique_configs: dict[tuple[str, int], RequiredVisualizer] = {} for ctx in compiled_contexts.values(): for required_visualizer in ctx.required_visualizers: if required_visualizer.key not in unique_configs: - unique_configs[required_visualizer.key] = ( - required_visualizer, - ctx.visualizer_config, - ) + unique_configs[required_visualizer.key] = required_visualizer - for required_visualizer, vis_config in unique_configs.values(): + for required_visualizer in unique_configs.values(): emit_launching_viewer( required_visualizer=required_visualizer, progress_queue=progress_queue, progress_context=progress_context, ) - yield orchestrator.get_or_create_visualizer( - required_visualizer.config, - vis_config, - ) + yield orchestrator.get_or_create_visualizer(required_visualizer.config) def rollback_failed_visualizer_bootstrap( diff --git a/openhcs/core/orchestrator/execution_result.py b/openhcs/core/orchestrator/execution_result.py index 585bd3eb31..fb063a2712 100644 --- a/openhcs/core/orchestrator/execution_result.py +++ b/openhcs/core/orchestrator/execution_result.py @@ -12,7 +12,10 @@ from typing import TYPE_CHECKING, ClassVar, Mapping from openhcs.constants.constants import Backend -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ( + ExecutingContext, + ProcessingContext, +) from openhcs.core.progress.types import ( AxisErrorProgressPhase, CancelledProgressPhase, @@ -88,7 +91,7 @@ class RuntimeExecutionObservation: @classmethod def from_completed_outputs( - cls, contexts: Mapping[str, ProcessingContext] + cls, contexts: Mapping[str, ExecutingContext] ) -> "RuntimeExecutionObservation": """Project saved facts independently of retained runtime pixel records.""" return cls(contexts=tuple( @@ -101,7 +104,7 @@ def from_completed_outputs( if not context.completed_step_outputs.is_empty )) - def merge_into(self, execution_contexts: Mapping[str, ProcessingContext]) -> None: + def merge_into(self, execution_contexts: Mapping[str, ExecutingContext]) -> None: """Merge returned runtime records into parent-owned compiled contexts.""" for context_observation in self.contexts: context = execution_contexts[context_observation.context_key] diff --git a/openhcs/core/orchestrator/orchestrator.py b/openhcs/core/orchestrator/orchestrator.py index f315b1961e..aa7e905482 100644 --- a/openhcs/core/orchestrator/orchestrator.py +++ b/openhcs/core/orchestrator/orchestrator.py @@ -19,7 +19,7 @@ from openhcs.core.metadata_cache import MetadataCache -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import CompilingContext from openhcs.core.input_workspace import InputWorkspacePreparationResult from openhcs.core.source_binding_context import SourceBindingContext from openhcs.core.pipeline.compiler import PipelineCompiler @@ -632,8 +632,8 @@ def get_results_path(self) -> Path: def create_context( self, axis_id: str, *, resolved_config: GlobalPipelineConfig | None = None - ) -> ProcessingContext: - """Creates a ProcessingContext for a given multiprocessing axis value.""" + ) -> CompilingContext: + """Start compiling the context of one multiprocessing-axis value.""" if not self.is_initialized(): raise RuntimeError( "Orchestrator must be initialized before calling create_context()." @@ -648,7 +648,7 @@ def create_context( effective_config = ( self.get_effective_config() if resolved_config is None else resolved_config ) - context = ProcessingContext( + context = CompilingContext( axis_id=axis_id, filemanager=self.filemanager, tiff_config=effective_config.tiff_config, diff --git a/openhcs/core/orchestrator/worker_execution.py b/openhcs/core/orchestrator/worker_execution.py index 0ca3555ccb..c1b7cce080 100644 --- a/openhcs/core/orchestrator/worker_execution.py +++ b/openhcs/core/orchestrator/worker_execution.py @@ -25,7 +25,10 @@ ) from openhcs.core.config import MultiprocessingStartMethod from openhcs.core.callable_contract import FunctionStepExecutionScope -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ( + CompiledContext, + ExecutingContext, +) from openhcs.core.function_step_transport import FunctionStepTransportAuthority from openhcs.core.native_threading import configure_native_thread_count from openhcs.core.runtime_profile import RuntimeProfileLogger @@ -155,7 +158,7 @@ def uses_fork_inherited_contexts(self) -> bool: def contexts_snapshot( self, execution_bundle: CompiledExecutionBundle, - ) -> Dict[str, ProcessingContext]: + ) -> Dict[str, CompiledContext]: """Return the context map consumed by lane planning for this mode.""" @abstractmethod @@ -164,7 +167,7 @@ def run_worker_lanes( *, pipeline_definition: List[AbstractStep], worker_lane_execution_plan: WorkerLaneExecutionPlan, - parent_contexts: Mapping[str, ProcessingContext], + parent_contexts: Mapping[str, ExecutingContext], ) -> Dict[str, ExecutionResult]: """Execute all lane work for this mode.""" @@ -204,7 +207,7 @@ def uses_fork_inherited_contexts(self) -> bool: def contexts_snapshot( self, execution_bundle: CompiledExecutionBundle, - ) -> Dict[str, ProcessingContext]: + ) -> Dict[str, CompiledContext]: return dict(execution_bundle.runtime_contexts) def run_worker_lanes( @@ -212,7 +215,7 @@ def run_worker_lanes( *, pipeline_definition: List[AbstractStep], worker_lane_execution_plan: WorkerLaneExecutionPlan, - parent_contexts: Mapping[str, ProcessingContext], + parent_contexts: Mapping[str, ExecutingContext], ) -> Dict[str, ExecutionResult]: lane_results = InlineWorkerLaneRunner(self.cancellation).run( pipeline_definition, @@ -297,7 +300,7 @@ def uses_fork_inherited_contexts(self) -> bool: def contexts_snapshot( self, execution_bundle: CompiledExecutionBundle, - ) -> Dict[str, ProcessingContext]: + ) -> Dict[str, CompiledContext]: return dict(execution_bundle.runtime_contexts) def install_execution_bundle( @@ -313,7 +316,7 @@ def run_worker_lanes( *, pipeline_definition: List[AbstractStep], worker_lane_execution_plan: WorkerLaneExecutionPlan, - parent_contexts: Mapping[str, ProcessingContext], + parent_contexts: Mapping[str, ExecutingContext], ) -> Dict[str, ExecutionResult]: return self._runner.run(worker_lane_execution_plan) @@ -344,7 +347,7 @@ def map_partition_invocations( def contexts_snapshot( self, execution_bundle: CompiledExecutionBundle, - ) -> Dict[str, ProcessingContext]: + ) -> Dict[str, CompiledContext]: return dict(execution_bundle.transport_contexts) def run_worker_lanes( @@ -352,7 +355,7 @@ def run_worker_lanes( *, pipeline_definition: List[AbstractStep], worker_lane_execution_plan: WorkerLaneExecutionPlan, - parent_contexts: Mapping[str, ProcessingContext], + parent_contexts: Mapping[str, ExecutingContext], ) -> Dict[str, ExecutionResult]: return PooledWorkerLaneRunner( self._executor, @@ -421,7 +424,7 @@ class ThreadedWorkerExecutorResources(PooledWorkerExecutorResources): def contexts_snapshot( self, execution_bundle: CompiledExecutionBundle, - ) -> Dict[str, ProcessingContext]: + ) -> Dict[str, CompiledContext]: return dict(execution_bundle.runtime_contexts) @@ -1046,7 +1049,7 @@ def run_worker_lanes( *, pipeline_definition: List[AbstractStep], worker_lane_execution_plan: WorkerLaneExecutionPlan, - parent_contexts: Mapping[str, ProcessingContext], + parent_contexts: Mapping[str, ExecutingContext], ) -> Dict[str, ExecutionResult]: execution_bundle = self._execution_bundle if execution_bundle is None: @@ -1180,7 +1183,7 @@ def run( self, pipeline_definition: List[AbstractStep], execution_plan: WorkerLaneExecutionPlan, - parent_contexts: Mapping[str, ProcessingContext], + parent_contexts: Mapping[str, ExecutingContext], ) -> Dict[str, ExecutionResult]: future_to_worker_slot = self._submit_lanes( pipeline_definition, @@ -1227,7 +1230,7 @@ def _collect_results( future_to_worker_slot: Mapping[concurrent.futures.Future, str], pipeline_definition: List[AbstractStep], execution_plan: WorkerLaneExecutionPlan, - parent_contexts: Mapping[str, ProcessingContext], + parent_contexts: Mapping[str, ExecutingContext], ) -> Dict[str, ExecutionResult]: execution_results: Dict[str, ExecutionResult] = {} lane_errors: list[Exception] = [] @@ -1315,7 +1318,7 @@ def _execute_axis_with_sequential_combinations( ) runtime_observations: list[RuntimeContextObservation] = [] - for context_key, frozen_context in axis_contexts: + for context_key, compiled_context in axis_contexts: if cancellation is not None: try: cancellation.raise_if_requested(f"before context {context_key}") @@ -1327,39 +1330,33 @@ def _execute_axis_with_sequential_combinations( contexts=tuple(runtime_observations), ), ) - runtime_store = frozen_context.runtime_value_store - execution_observation_cursor = runtime_store.observation_cursor() + context = compiled_context.executing(lane_context) try: result = _execute_single_axis_static( pipeline_definition, - frozen_context, + context, lane_context, context_key=context_key, cancellation=cancellation, runtime_observation_mode=runtime_observation_mode, ) - observed_records = runtime_store.observed_values_after( - execution_observation_cursor - ) observation = RuntimeContextObservation.from_context( context_key=context_key, - context=frozen_context, - records=observed_records, + context=context, + records=tuple(context.runtime_value_store.observed_values), runtime_observation_mode=runtime_observation_mode, outputs=StepExecutionObservation.combine( item.outputs for item in result.runtime_observation.contexts ), ) finally: - # This cache is context-local even when lanes share a process. - # Required records and table projections now belong to the observation. - frozen_context.release_execution_image_cache() + # Required records and table projections now belong to the + # observation; the execution context and its stores end here. if release_axis_resources: - _release_runtime_resources((frozen_context,), owner=f"axis {axis_id}") - frozen_context.runtime_value_store.clear() + _release_runtime_resources((context,), owner=f"axis {axis_id}") + del context if observation.records or not observation.outputs.is_empty: runtime_observations.append(observation) - del observed_records if not result.is_success(): logger.error( @@ -1398,7 +1395,7 @@ def _execute_axis_with_sequential_combinations( def _release_runtime_resources( - contexts: Iterable[ProcessingContext], + contexts: Iterable[CompiledContext], *, owner: str, ) -> None: @@ -1432,7 +1429,7 @@ def _release_runtime_resources( def _completed_axis_step_count( - context: ProcessingContext, + context: CompiledContext, total_steps: int, ) -> int: """Return the pipeline position where terminal dataset-scoped processing begins.""" @@ -1449,31 +1446,24 @@ def _completed_axis_step_count( def _execute_single_axis_static( pipeline_definition: List[AbstractStep], - frozen_context: ProcessingContext, + context: ExecutingContext, lane_context: WorkerLaneExecutionContext, *, context_key: str, cancellation: ExecutionCancellationSignal | None = None, runtime_observation_mode: type[RuntimeObservationMode] = MergeAllRuntimeRecords, ) -> ExecutionResult: - """Execute one frozen axis context against the compiled pipeline.""" + """Execute one compiled axis context against the compiled pipeline.""" - axis_id = frozen_context.axis_id + axis_id = context.axis_id total_steps = len(pipeline_definition) - if not frozen_context.is_frozen(): - error_msg = f"Context for axis {axis_id} is not frozen before execution" - logger.error(error_msg) - raise RuntimeError(error_msg) - if not pipeline_definition: error_msg = f"Empty pipeline_definition for axis {axis_id}" logger.error(error_msg) raise RuntimeError(error_msg) - frozen_context.bind_execution_runtime(lane_context) - lane_context.install_debug_sink(frozen_context) - runtime_value_store = frozen_context.runtime_value_store + runtime_value_store = context.runtime_value_store # Full-value observations, debug replay and viewer histories own their # original retention contract. Ordinary execution can retire dead inputs # only after the step's output/progress consumers have completed. @@ -1482,14 +1472,14 @@ def _execute_single_axis_static( and isinstance(lane_context.debug_execution_policy, NoOpDebugExecutionPolicy) and not any( plan.visualize or plan.streaming_configs - for plan in frozen_context.step_plans.values() + for plan in context.step_plans.values() ) ) checkpoint_refs = ( frozenset( output.ref().for_plan_type(ArtifactInputPlan) - for plan in frozen_context.step_plans.values() - if plan.requires_main_flow_checkpoint(frozen_context.step_plans) + for plan in context.step_plans.values() + if plan.requires_main_flow_checkpoint(context.step_plans) for output in plan.artifact_outputs.values() ) if release_consumed @@ -1500,7 +1490,7 @@ def _execute_single_axis_static( for step_index, step in enumerate(pipeline_definition): if cancellation is not None: cancellation.raise_if_requested(f"before step {step_index + 1}") - step_plan = frozen_context.step_plans[step_index] + step_plan = context.step_plans[step_index] compiled_pattern = step_plan.compiled_function_pattern if ( compiled_pattern is not None @@ -1517,7 +1507,7 @@ def _execute_single_axis_static( step_index=step_index, step_name=step_name, step_scope_id=step_plan.step_scope_id, - context=frozen_context, + context=context, artifact_outputs=step_plan.artifact_outputs, ) observed_records = runtime_value_store.observed_values_after( @@ -1525,14 +1515,14 @@ def _execute_single_axis_static( ) reused_outputs = preview_reused_step_outputs( step_plan, - frozen_context, + context, observed_records, ) runtime_progress_context = _runtime_observation_progress_context( observed_records, materialized_locations_by_address=reused_outputs.materialized_locations_by_address, ) - frozen_context.record_completed_step_outputs(reused_outputs) + context.record_completed_step_outputs(reused_outputs) lane_context.emit( axis_id=axis_id, step_name=step_name, @@ -1557,8 +1547,8 @@ def _execute_single_axis_static( ) observation_cursor = runtime_value_store.observation_cursor() - step_observation = step.process(frozen_context, step_index) - frozen_context.record_completed_step_outputs(step_observation) + step_observation = step.process(context, step_index) + context.record_completed_step_outputs(step_observation) observed_records = runtime_value_store.observed_values_after( observation_cursor ) @@ -1588,9 +1578,9 @@ def _execute_single_axis_static( step_plan.future_artifact_inputs | checkpoint_refs, runtime_observation_mode.retain_records( runtime_value_store.observed_values, - frozen_context, + context, ), - filemanager=frozen_context.filemanager, + filemanager=context.filemanager, ) except ExecutionCancelledError as exc: @@ -1598,7 +1588,7 @@ def _execute_single_axis_static( axis_id=axis_id, error_message=str(exc), runtime_observation=RuntimeExecutionObservation.from_completed_outputs( - {context_key: frozen_context} + {context_key: context} ), ) except Exception as exc: @@ -1610,14 +1600,14 @@ def _execute_single_axis_static( failed_combination=context_key, error_message=str(exc), runtime_observation=RuntimeExecutionObservation.from_completed_outputs( - {context_key: frozen_context} + {context_key: context} ), ) return ExecutionSucceeded( axis_id=axis_id, runtime_observation=RuntimeExecutionObservation.from_completed_outputs( - {context_key: frozen_context} + {context_key: context} ), ) diff --git a/openhcs/core/orchestrator/worker_lanes.py b/openhcs/core/orchestrator/worker_lanes.py index 14196e32e7..1af991ad90 100644 --- a/openhcs/core/orchestrator/worker_lanes.py +++ b/openhcs/core/orchestrator/worker_lanes.py @@ -9,7 +9,7 @@ from python_introspect import project_dataclass from openhcs.core.compiled_execution import CompiledExecutionBundle -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import CompiledContext from openhcs.core.debug import ( DebugExecutionContext, DebugExecutionPolicy, @@ -18,7 +18,7 @@ from openhcs.core.orchestrator.execution_result import RuntimeObservationMode from openhcs.core.progress import ProgressExecutionContext -TransportAxisContexts: TypeAlias = List[tuple[str, ProcessingContext]] +TransportAxisContexts: TypeAlias = List[tuple[str, CompiledContext]] ForkAxisContextKeys: TypeAlias = List[str] LaneAxisContextPayload: TypeAlias = TransportAxisContexts | ForkAxisContextKeys WorkerLaneAxisContexts: TypeAlias = List[tuple[str, LaneAxisContextPayload]] @@ -150,7 +150,7 @@ def __init__( def plan( self, - contexts_snapshot: Mapping[str, ProcessingContext], + contexts_snapshot: Mapping[str, CompiledContext], worker_assignments: Mapping[str, Sequence[str]], ) -> WorkerAssignmentPlan: contexts_by_axis = self._contexts_by_axis(contexts_snapshot) @@ -172,7 +172,7 @@ def plan( def _contexts_by_axis( self, - contexts_snapshot: Mapping[str, ProcessingContext], + contexts_snapshot: Mapping[str, CompiledContext], ) -> Dict[str, List[str]]: contexts_by_axis: dict[str, list[str]] = defaultdict(list) for context_key, context in contexts_snapshot.items(): @@ -242,7 +242,7 @@ def _axis_to_worker( def _lane_axis_contexts( self, - contexts_snapshot: Mapping[str, ProcessingContext], + contexts_snapshot: Mapping[str, CompiledContext], contexts_by_axis: Mapping[str, List[str]], worker_assignments: Mapping[str, List[str]], axis_to_worker: Mapping[str, str], diff --git a/openhcs/core/pipeline/compilation_session.py b/openhcs/core/pipeline/compilation_session.py index fd7d7a870d..776357a4ed 100644 --- a/openhcs/core/pipeline/compilation_session.py +++ b/openhcs/core/pipeline/compilation_session.py @@ -9,7 +9,7 @@ from objectstate import DataclassFieldAccess from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import CompilingContext from openhcs.core.source_metadata import SourceMetadataFields, SourceMetadataMapping from openhcs.core.source_workspace_projection import VirtualWorkspaceSourceProjection from openhcs.core.steps.abstract import AbstractStep @@ -52,7 +52,7 @@ def __post_init__(self) -> None: ) @classmethod - def from_context(cls, context: ProcessingContext) -> "CompilationDatasetScope": + def from_context(cls, context: CompilingContext) -> "CompilationDatasetScope": if context.dataset_root is None: raise ValueError("Compilation plate scope requires context.dataset_root.") return cls(Path(context.dataset_root)) @@ -407,7 +407,7 @@ class CompilationSession: compiled-plan map for one axis or sequential-combination context. """ - context: ProcessingContext + context: CompilingContext pipeline: ResolvedPipelineDefinition orchestrator: "PipelineOrchestrator" global_config: "GlobalPipelineConfig" @@ -426,7 +426,7 @@ class CompilationSession: def from_context( cls, *, - context: ProcessingContext, + context: CompilingContext, pipeline: ResolvedPipelineDefinition, orchestrator: "PipelineOrchestrator", global_config: "GlobalPipelineConfig", diff --git a/openhcs/core/pipeline/compiler.py b/openhcs/core/pipeline/compiler.py index 1620eade26..c87f04ec2a 100644 --- a/openhcs/core/pipeline/compiler.py +++ b/openhcs/core/pipeline/compiler.py @@ -37,7 +37,8 @@ PayloadAxisRequirement, ) from openhcs.core.context.processing_context import ( - ProcessingContext, + CompiledContext, + CompilingContext, RequiredVisualizer, ) from openhcs.core.config import ( @@ -166,7 +167,7 @@ class AxisCompilationRequest: enable_visualizer_override: bool is_zmq_execution: bool - def context_for(self, axis_id: str) -> ProcessingContext: + def context_for(self, axis_id: str) -> CompilingContext: context = self.orchestrator.create_context( axis_id, resolved_config=self.global_config ) @@ -196,7 +197,7 @@ class PipelineCompiler: @staticmethod def initialize_step_plans_for_context( - context: ProcessingContext, + context: CompilingContext, pipeline: ResolvedPipelineDefinition, orchestrator, global_config: "GlobalPipelineConfig", @@ -223,8 +224,6 @@ def initialize_step_plans_for_context( Returns: The axis-scoped compilation session used by all downstream stages. """ - PipelineCompiler._assert_context_mutable_for_planning(context) - context.visualizer_config = None logger.debug("Using pipeline-resolved steps for context %s", context.axis_id) session = CompilationSession.from_context( context=context, @@ -247,15 +246,6 @@ def initialize_step_plans_for_context( PipelineCompiler._collect_streaming_configs(session) return session - @staticmethod - def _assert_context_mutable_for_planning(context: ProcessingContext) -> None: - if context.is_frozen(): - raise AttributeError( - "Cannot initialize step plans in a frozen ProcessingContext." - ) - if context.step_plans is None: - context.step_plans = {} - @staticmethod def _missing_plan_fields( plan: CompiledStepPlan, @@ -533,10 +523,6 @@ def plan_materialization_flags( by calling MaterializationFlagPlanner. """ context = session.context - if context.is_frozen(): - raise AttributeError( - "Cannot plan materialization flags in a frozen ProcessingContext." - ) if not context.step_plans: logger.warning( "step_plans is empty in context for materialization planning. This may be valid if pipeline is empty." @@ -644,7 +630,7 @@ def validate_sequential_components_compatibility( @staticmethod def analyze_pipeline_sequential_mode( - context: ProcessingContext, + context: CompilingContext, global_config: GlobalPipelineConfig, orchestrator: PipelineOrchestrator, ) -> None: @@ -657,10 +643,6 @@ def analyze_pipeline_sequential_mode( global_config: GlobalPipelineConfig containing SequentialProcessingConfig orchestrator: PipelineOrchestrator with microscope handler for pattern discovery """ - if context.is_frozen(): - raise AttributeError( - "Cannot analyze pipeline sequential mode in a frozen ProcessingContext." - ) # Get pipeline-level sequential processing config seq_config = global_config.sequential_processing_config @@ -735,10 +717,6 @@ def validate_memory_contracts(session: CompilationSession) -> None: orchestrator: Optional orchestrator for dict pattern key validation """ context = session.context - if context.is_frozen(): - raise AttributeError( - "Cannot validate memory contracts in a frozen ProcessingContext." - ) FuncStepContractValidator.validate_pipeline( steps=session.pipeline.steps, @@ -772,10 +750,6 @@ def validate_memory_contracts(session: CompilationSession) -> None: def assign_framework_device_resources(session: CompilationSession) -> None: """Resolve each step's declaration-derived framework device footprint.""" context = session.context - if context.is_frozen(): - raise AttributeError( - "Cannot assign GPU resources in a frozen ProcessingContext." - ) assign_framework_devices(context.step_plans) @@ -797,10 +771,6 @@ def apply_global_visualizer_override( Applies global visualizer override to all step_plans in the context. """ context = session.context - if context.is_frozen(): - raise AttributeError( - "Cannot apply visualizer override in a frozen ProcessingContext." - ) if global_enable_visualizer: if not context.step_plans: @@ -996,10 +966,10 @@ def _resolve_global_step_axis_filters( def _compile_axis_values( request: AxisCompilationRequest, axis_values: Sequence[str], - ) -> Dict[str, ProcessingContext]: + ) -> Dict[str, CompiledContext]: # The final declaration admits providers before any axis-specific work. request.pipeline.invocation_contract_provider - compiled_contexts: Dict[str, ProcessingContext] = {} + compiled_contexts: Dict[str, CompiledContext] = {} responsible_axis_value = sorted(axis_values)[0] total_axis_values = len(axis_values) for completed, axis_id in enumerate(axis_values, start=1): @@ -1024,7 +994,7 @@ def _compile_axis_value( request: AxisCompilationRequest, axis_id: str, metadata_writer: bool, - ) -> Dict[str, ProcessingContext]: + ) -> Dict[str, CompiledContext]: context = request.context_for(axis_id) session = PipelineCompiler.build_initialize_axis_session( request, @@ -1057,7 +1027,7 @@ def _compile_axis_value( @staticmethod def build_initialize_axis_session( request: AxisCompilationRequest, - context: ProcessingContext, + context: CompilingContext, metadata_writer: bool, ) -> CompilationSession: return PipelineCompiler.initialize_step_plans_for_context( @@ -1076,11 +1046,11 @@ def build_initialize_axis_session( def _compile_sequential_axis_contexts( *, request: AxisCompilationRequest, - temp_context: ProcessingContext, + temp_context: CompilingContext, axis_id: str, metadata_writer: bool, - ) -> Dict[str, ProcessingContext]: - compiled_contexts: Dict[str, ProcessingContext] = {} + ) -> Dict[str, CompiledContext]: + compiled_contexts: Dict[str, CompiledContext] = {} combinations = temp_context.pipeline_sequential_combinations for combo_idx, combo in enumerate(combinations): context = request.context_for(axis_id) @@ -1100,8 +1070,7 @@ def _compile_sequential_axis_contexts( session, enable_visualizer_override=request.enable_visualizer_override, ) - context.freeze() - compiled_contexts[f"{axis_id}__combo_{combo_idx}"] = context + compiled_contexts[f"{axis_id}__combo_{combo_idx}"] = context.freeze() return compiled_contexts @staticmethod @@ -1110,7 +1079,7 @@ def _compile_single_axis_context( *, materialization_planner: MaterializationFlagPlanner, enable_visualizer_override: bool, - ) -> ProcessingContext: + ) -> CompiledContext: """Finish a nonsequential axis using its validated initial plan.""" context = session.context PipelineCompiler.declare_zarr_stores(session) @@ -1119,8 +1088,7 @@ def _compile_single_axis_context( session, enable_visualizer_override=enable_visualizer_override, ) - context.freeze() - return context + return context.freeze() @staticmethod def _run_post_plan_compile_stages( @@ -1468,7 +1436,7 @@ def _emit_axis_compile_progress( def _finalize_compilation( orchestrator, pipeline_definition: List[AbstractStep], - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, CompiledContext], effective_config: GlobalPipelineConfig, ) -> None: PipelineCompiler._log_path_planning_summary(compiled_contexts) @@ -1488,7 +1456,7 @@ def _finalize_compilation( @staticmethod def _log_path_planning_summary( - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, CompiledContext], ) -> None: if not compiled_contexts: return diff --git a/openhcs/core/post_execute.py b/openhcs/core/post_execute.py index 340025b598..42f3ee2e8c 100644 --- a/openhcs/core/post_execute.py +++ b/openhcs/core/post_execute.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.config import GlobalPipelineConfig - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.orchestrator.execution_result import ( ExecutionResult, RuntimeExecutionObservation, @@ -53,7 +53,7 @@ def bind(cls, global_config: "GlobalPipelineConfig") -> "PostExecuteHook": def observe_saved_outputs( self, - context: "ProcessingContext", + context: "ExecutingContext", plan: "CompiledStepPlan", saved: "MaterializedRuntimeArtifact", ) -> object | None: @@ -63,7 +63,7 @@ def observe_saved_outputs( def observe_reused_outputs( self, - context: "ProcessingContext", + context: "ExecutingContext", plan: "CompiledStepPlan", materialization: "RuntimeArtifactMaterialization", outputs: tuple["Output", ...], @@ -85,7 +85,7 @@ def combine(cls, observations: Iterable[object]) -> object | None: @abstractmethod def run( self, - compiled_contexts: Mapping[str, "ProcessingContext"], + compiled_contexts: Mapping[str, "ExecutingContext"], observation: object | None, ) -> None: """Do this hook's work after every partition succeeded.""" @@ -101,7 +101,7 @@ def bind_all(global_config: "GlobalPipelineConfig") -> tuple["PostExecuteHook", @staticmethod def observe_saved( - context: "ProcessingContext", + context: "ExecutingContext", plan: "CompiledStepPlan", saved: "MaterializedRuntimeArtifact", ) -> HookObservations: @@ -112,7 +112,7 @@ def observe_saved( @staticmethod def observe_reused( - context: "ProcessingContext", + context: "ExecutingContext", plan: "CompiledStepPlan", materialization: "RuntimeArtifactMaterialization", outputs: tuple["Output", ...], @@ -135,7 +135,7 @@ def combine_all(observations: Iterable[HookObservations]) -> HookObservations: @staticmethod def run_all( - compiled_contexts: Mapping[str, "ProcessingContext"], + compiled_contexts: Mapping[str, "ExecutingContext"], execution_results: Mapping[str, "ExecutionResult"], dataset_runtime_observation: "RuntimeExecutionObservation", ) -> None: diff --git a/openhcs/core/runtime_adapters.py b/openhcs/core/runtime_adapters.py index 68960a451e..586ad99e3f 100644 --- a/openhcs/core/runtime_adapters.py +++ b/openhcs/core/runtime_adapters.py @@ -44,7 +44,7 @@ if TYPE_CHECKING: from openhcs.core.callable_contract import CallableContract - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.runtime_stores import RuntimeArtifactInput from openhcs.core.steps.function_runtime import FunctionCoreExecutor @@ -65,7 +65,7 @@ class RuntimeImageExecutionContext: class RuntimeAdapterRequest: """Runtime data needed to build an invocation-scoped adapter.""" - context: "ProcessingContext" + context: "ExecutingContext" callable_contract: "CallableContract | None" = None source_payload: object | None = None artifact_inputs: Mapping[ diff --git a/openhcs/core/runtime_batch_contracts.py b/openhcs/core/runtime_batch_contracts.py index 48f300df7e..f076a71478 100644 --- a/openhcs/core/runtime_batch_contracts.py +++ b/openhcs/core/runtime_batch_contracts.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: from openhcs.core.processing_contracts import ProcessingContract from openhcs.core.runtime_stores import RuntimeArtifactBatch - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext F = TypeVar("F", bound=Callable) @@ -124,13 +124,13 @@ class RuntimeArtifactPartitionBatchRequest: """Parent-admitted artifact batch mapped through existing worker resources. The declared executor owns partition eligibility and global reductions. - ProcessingContext stays in the parent and is never part of worker requests. + ExecutingContext stays in the parent and is never part of worker requests. """ func: Callable[..., object] artifact_batch: RuntimeArtifactBatch kwargs: Mapping[str, object] - runtime_context: ProcessingContext | None + runtime_context: ExecutingContext | None map_partition_invocations: Callable[ [Callable[[object], object], Sequence[object]], tuple[object, ...] ] @@ -142,7 +142,7 @@ def from_contract( *, artifact_batch: RuntimeArtifactBatch, kwargs: Mapping[str, object], - runtime_context: ProcessingContext | None, + runtime_context: ExecutingContext | None, map_partition_invocations: Callable[ [Callable[[object], object], Sequence[object]], tuple[object, ...] ], diff --git a/openhcs/core/runtime_execution_validation.py b/openhcs/core/runtime_execution_validation.py index 5ecde8aed8..6c0cba1f4f 100644 --- a/openhcs/core/runtime_execution_validation.py +++ b/openhcs/core/runtime_execution_validation.py @@ -12,7 +12,10 @@ from openhcs.core.artifacts.specs import ArtifactSpec from openhcs.core.artifacts.kinds import ArtifactType -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ( + ExecutingContext, + ProcessingContext, +) from openhcs.core.orchestrator.execution_result import RuntimeExecutionObservation from openhcs.core.runtime_exports import ( RuntimeExportExpectation, @@ -126,9 +129,9 @@ class RuntimeArtifactExecutionExpectation: artifact_viewer: tuple[RuntimeArtifactViewerExpectation, ...] = () @classmethod - def from_compiled_contexts( + def from_executed_contexts( cls, - compiled_contexts: Mapping[str, ProcessingContext], + executed_contexts: Mapping[str, ExecutingContext], ) -> "RuntimeArtifactExecutionExpectation": """Derive execution expectations from compiler-owned output plans.""" output_specs = tuple( @@ -140,7 +143,7 @@ def from_compiled_contexts( viewer_streaming=output.viewer_streaming, sidecar_role=output.sidecar_role, ) - for context in compiled_contexts.values() + for context in executed_contexts.values() for step_plan in context.step_plans.values() for output in step_plan.artifact_outputs.values() ) @@ -148,7 +151,7 @@ def from_compiled_contexts( return cls( artifact_kinds=frozenset(spec.artifact_type for spec in output_specs), exports=RuntimeExportExpectation.from_output_specs(output_specs), - artifact_viewer=runtime_artifact_viewer_expectations(compiled_contexts), + artifact_viewer=runtime_artifact_viewer_expectations(executed_contexts), axis_expectations=tuple( RuntimeArtifactAxisExpectation( axis_id=str(axis_id), @@ -159,7 +162,7 @@ def from_compiled_contexts( for output in plan.artifact_outputs.values() ), ) - for axis_id, context in compiled_contexts.items() + for axis_id, context in executed_contexts.items() ), ) @@ -207,7 +210,7 @@ def __post_init__(self) -> None: def runtime_artifact_viewer_expectations( - compiled_contexts: Mapping[str, ProcessingContext], + executed_contexts: Mapping[str, ExecutingContext], ) -> tuple[RuntimeArtifactViewerExpectation, ...]: """Derive artifact-only viewer expectations from compiled plans and records.""" @@ -215,7 +218,7 @@ def runtime_artifact_viewer_expectations( StreamProducerIdentity, list[RuntimeArtifactViewerPayloadExpectation], ] = {} - for context in compiled_contexts.values(): + for context in executed_contexts.values(): for plan in context.step_plans.values(): if not plan.streaming_configs or not plan.owns_runtime_outputs: continue @@ -299,7 +302,7 @@ class RuntimeArtifactExecutionObservation: @classmethod def from_contexts( cls, - execution_contexts: Mapping[str, ProcessingContext], + execution_contexts: Mapping[str, ExecutingContext], *, runtime_observations: Sequence[RuntimeExecutionObservation], ) -> "RuntimeArtifactExecutionObservation": @@ -357,7 +360,7 @@ def record_counts_by_axis(self) -> Mapping[str, Mapping[type[ArtifactType], int] def runtime_records_by_axis( - execution_contexts: Mapping[str, ProcessingContext], + execution_contexts: Mapping[str, ExecutingContext], ) -> Mapping[str, tuple[StoredRuntimeValue, ...]]: """Return stored runtime records from compiled execution contexts.""" records_by_axis: dict[str, tuple[StoredRuntimeValue, ...]] = {} diff --git a/openhcs/core/source_binding_selection.py b/openhcs/core/source_binding_selection.py index 77264b9db3..a8ceab7d81 100644 --- a/openhcs/core/source_binding_selection.py +++ b/openhcs/core/source_binding_selection.py @@ -71,7 +71,7 @@ from openhcs.core.runtime_adapters import RuntimeAdapterRequest from openhcs.core.runtime_source_binding_cache import RuntimeSourceResolutionSnapshot from polystore.filemanager import FileManager - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.dataset_sources.interfaces import FilenameParser @@ -1608,7 +1608,7 @@ class SourceUniverseRequest(metaclass=AutoRegisterMeta): __skip_if_no_key__ = True universe_request_kind: ClassVar[str | None] = None - context: "ProcessingContext" + context: "ExecutingContext" plan: "CompiledStepPlan" matching_files: tuple[str, ...] source_backend: Backend @@ -1618,7 +1618,7 @@ class SourceUniverseRequest(metaclass=AutoRegisterMeta): def from_context( cls, *, - context: "ProcessingContext", + context: "ExecutingContext", plan: "CompiledStepPlan", matching_files: Sequence[str], source_projection: VirtualWorkspaceSourceProjection | None, diff --git a/openhcs/core/source_workspace_projection.py b/openhcs/core/source_workspace_projection.py index 5f67a165c1..e7ad8ca308 100644 --- a/openhcs/core/source_workspace_projection.py +++ b/openhcs/core/source_workspace_projection.py @@ -47,7 +47,7 @@ from openhcs.core.runtime_image_values import ImagePayload if TYPE_CHECKING: - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.dataset_sources.interfaces import MetadataHandler from openhcs.core.vfs_protocol import FileManagerLike from polystore.filemanager import FileManager @@ -846,7 +846,7 @@ class WorkspaceSourceProjections: @classmethod def from_context( cls, - context: "ProcessingContext", + context: "ExecutingContext", *, cache: VirtualWorkspaceSourceProjectionCache | None = None, ) -> "WorkspaceSourceProjections": @@ -879,7 +879,7 @@ def from_dataset_metadata( source_bindings=source_bindings, ) - def is_bound_to_context(self, context: "ProcessingContext") -> bool: + def is_bound_to_context(self, context: "ExecutingContext") -> bool: """Compare actual owners, never identities of already-released objects.""" if not ( self.dataset_root == Path(context.dataset_root) @@ -971,9 +971,9 @@ class RuntimeWorkspaceSourceProjections( ): """Read completed same-plate outputs from their execution observation owner.""" - context: "ProcessingContext" = field(kw_only=True, compare=False, repr=False) + context: "ExecutingContext" = field(kw_only=True, compare=False, repr=False) - def is_bound_to_context(self, context: "ProcessingContext") -> bool: + def is_bound_to_context(self, context: "ExecutingContext") -> bool: return self.context is context and super( RuntimeWorkspaceSourceProjections, self ).is_bound_to_context(context) diff --git a/openhcs/core/steps/abstract.py b/openhcs/core/steps/abstract.py index 2e891d1379..891d62308c 100644 --- a/openhcs/core/steps/abstract.py +++ b/openhcs/core/steps/abstract.py @@ -36,13 +36,13 @@ StoredRuntimeValue, ) -# ProcessingContext is used in type hints +# ExecutingContext is used in type hints if TYPE_CHECKING: from openhcs.core.steps.function_outputs import OpenHCSMetadataTarget from openhcs.core.virtual_workspace_metadata import ( VirtualWorkspaceSourceProjectionEntries, ) - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.post_execute import HookObservations @@ -197,7 +197,7 @@ class AbstractStep(abc.ABC): like name, input/output memory types, etc.). After compilation, these attributes are stripped by the StepAttributeStripper, and the step instances become stateless shells. During execution, steps operate solely based on the - ProcessingContext (which is frozen) and their specific plan within + ExecutingContext (which is frozen) and their specific plan within context.step_plans. Input Source Control: @@ -371,7 +371,7 @@ def __init__( @abc.abstractmethod def process( self, - context: "ProcessingContext", + context: "ExecutingContext", step_index: int, ) -> "StepExecutionObservation": """ @@ -386,7 +386,7 @@ def process( performed during this step execution. Args: - context: The frozen ProcessingContext containing all required fields, + context: The frozen ExecutingContext containing all required fields, including step_plans and filemanager. """ raise NotImplementedError( diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index 3aaf5669d0..97e70b85b9 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -68,7 +68,7 @@ if TYPE_CHECKING: from polystore.filemanager import FileManager - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext logger = logging.getLogger(__name__) @@ -84,7 +84,7 @@ class ArtifactMaterializationTargetPlan(ABC, metaclass=AutoRegisterMeta): @classmethod def materialize( cls, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> tuple[MaterializedRuntimeArtifact, ...]: if not plan.artifact_outputs: @@ -126,7 +126,7 @@ def materialize_outputs( self, filemanager: "FileManager", plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", ) -> tuple[MaterializedRuntimeArtifact, ...]: """Save each exact artifact batch once and return its successful outputs.""" saved_materializations = [] @@ -209,7 +209,7 @@ def materialize_outputs( def streaming_viewer_surfaces( self, plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", materialization: "RuntimeArtifactMaterialization", ) -> Mapping[str, StreamingViewerSurface]: """Capture enabled destinations for this artifact at the write boundary.""" @@ -260,7 +260,7 @@ def backend_kwargs( streaming_viewer_surfaces: Mapping[str, StreamingViewerSurface], fallback_source_identity: SourceImageIdentity | None, producer_identity: StreamProducerIdentity, - context: "ProcessingContext", + context: "ExecutingContext", filemanager: "FileManager", images_dir: str, stream_output_paths: tuple[str, ...], @@ -324,7 +324,7 @@ def backend_kwargs( return result @abstractmethod - def persistent_backend_kwargs(self, context: "ProcessingContext") -> BackendKwargs: + def persistent_backend_kwargs(self, context: "ExecutingContext") -> BackendKwargs: """Return persistent materialization backends owned by this policy.""" @@ -335,7 +335,7 @@ class PersistentArtifactMaterializationTargetPlan(ArtifactMaterializationTargetP target_key = "persistent" backend: str - def persistent_backend_kwargs(self, context: "ProcessingContext") -> BackendKwargs: + def persistent_backend_kwargs(self, context: "ExecutingContext") -> BackendKwargs: return { self.backend: RawBackendKwargs( tiff_config=( @@ -350,7 +350,7 @@ class StreamingOnlyArtifactMaterializationTargetPlan(ArtifactMaterializationTarg target_key = "streaming_only" - def persistent_backend_kwargs(self, context: "ProcessingContext") -> BackendKwargs: + def persistent_backend_kwargs(self, context: "ExecutingContext") -> BackendKwargs: return {} @@ -570,7 +570,7 @@ def _analysis_identity( cls, output_key: str, plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", dict_key: str | None = None, artifact_path: str | None = None, record: StoredRuntimeValue | None = None, @@ -753,7 +753,7 @@ def record_metadata_with_runtime_scope( @classmethod def _record_source_identity( cls, - context: "ProcessingContext", + context: "ExecutingContext", plan: CompiledStepPlan, record: StoredRuntimeValue, materialization_spec: MaterializationSpec, @@ -834,7 +834,7 @@ def _record_source_identity( @staticmethod def source_identity_for_path( - context: "ProcessingContext", + context: "ExecutingContext", path: str | Path, ) -> SourceImageIdentity | None: """Return microscope source identity for a source path when available.""" @@ -869,7 +869,7 @@ def from_record( output_plan: ArtifactOutputPlan, record: StoredRuntimeValue, plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", ) -> "RuntimeArtifactMaterialization": """Build one materialization from an exact compiled output record.""" spec = output_plan.materialization @@ -970,7 +970,7 @@ def from_record( def outputs( self, plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", *, output_path_filter: Callable[[Path], bool] | None = None, ) -> tuple[Output, ...]: @@ -992,7 +992,7 @@ def outputs( def reused_observation( self, plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", ) -> StepExecutionObservation: """Report historical debug destinations and read their retained CSV text.""" from openhcs.core.post_execute import PostExecuteHook @@ -1028,7 +1028,7 @@ def reused_observation( def viewer_outputs( self, plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", ) -> tuple[Output, ...]: """Return exact writer outputs accepted by this step's viewer backends.""" surfaces = ( @@ -1053,7 +1053,7 @@ def viewer_outputs( def runtime_artifact_materializations( plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", ) -> tuple[RuntimeArtifactMaterialization, ...]: """Derive actual materializations from compiled outputs and runtime values.""" @@ -1080,7 +1080,7 @@ def runtime_artifact_materializations( def runtime_artifact_materializations_from_records( plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", records: tuple[StoredRuntimeValue, ...], ) -> tuple[RuntimeArtifactMaterialization, ...]: """Derive materializations from one caller-owned runtime observation.""" @@ -1116,7 +1116,7 @@ def runtime_artifact_materializations_from_records( def preview_reused_step_outputs( plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", records: tuple[StoredRuntimeValue, ...], ) -> StepExecutionObservation: """Project explicitly reused historical outputs once, independently of new saves.""" @@ -1133,7 +1133,7 @@ def preview_reused_step_outputs( def planned_materialization_preview( *, - context: "ProcessingContext", + context: "ExecutingContext", plan: CompiledStepPlan, output_key: str, output_plan: ArtifactOutputPlan, @@ -1169,7 +1169,7 @@ def planned_materialization_preview( def _planned_materialization_path( *, - context: "ProcessingContext", + context: "ExecutingContext", plan: CompiledStepPlan, output_key: str, output_plan: ArtifactOutputPlan, @@ -1201,7 +1201,7 @@ class MaterializedRuntimeArtifact(SavedMaterializationOutputs): def observation( self, plan: CompiledStepPlan, - context: "ProcessingContext", + context: "ExecutingContext", ) -> StepExecutionObservation: target = plan.runtime_artifact_materialization if not target.has_persistent_target: diff --git a/openhcs/core/steps/function_execution.py b/openhcs/core/steps/function_execution.py index 45c6668675..a0b9017836 100644 --- a/openhcs/core/steps/function_execution.py +++ b/openhcs/core/steps/function_execution.py @@ -22,7 +22,7 @@ from openhcs.core.runtime_profile import RuntimeProfileLogger, RuntimeProfileFieldValue from openhcs.core.callable_contract import ImagePayloadConsumption from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.function_patterns import ( CompiledFunctionGroup, FunctionGroupKey, @@ -123,7 +123,7 @@ def filter_pattern_list( class FunctionStepExecutor: """Run one compiled FunctionStep plan for one multiprocessing axis.""" - def __init__(self, context: ProcessingContext, step_index: int) -> None: + def __init__(self, context: ExecutingContext, step_index: int) -> None: self.context = context compiled_plan = context.step_plans[step_index] if not isinstance(compiled_plan, CompiledStepPlan): @@ -538,7 +538,7 @@ def record_runtime_profile( @classmethod def execute( cls, - context: ProcessingContext, + context: ExecutingContext, step_index: int, ) -> StepExecutionObservation: step_name = f"step_{step_index}" diff --git a/openhcs/core/steps/function_io.py b/openhcs/core/steps/function_io.py index 32e0212ee1..082a2b3de9 100644 --- a/openhcs/core/steps/function_io.py +++ b/openhcs/core/steps/function_io.py @@ -34,7 +34,7 @@ from polystore.filemanager import FileManager from openhcs.core.config import ZarrConfig - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.dataset_sources.source import DatasetSource logger = logging.getLogger(__name__) @@ -267,7 +267,7 @@ def save_materialized_data( materialized_paths: Sequence[str], materialized_backend: str, zarr_config: ZarrBackendConfig | None, - context: ProcessingContext, + context: ExecutingContext, axis_id: str, *, output_identities: Sequence[FunctionOutputIdentity] = (), @@ -374,7 +374,7 @@ def update_metadata_for_zarr_conversion( dataset_root: Path, original_subdir: str, zarr_subdir: str | None, - context: ProcessingContext, + context: ExecutingContext, ) -> None: """Update OpenHCS metadata after a Zarr input conversion.""" from openhcs.core.virtual_workspace_metadata import ( diff --git a/openhcs/core/steps/function_output_manifest.py b/openhcs/core/steps/function_output_manifest.py index ff79b3da72..85285ad2b9 100644 --- a/openhcs/core/steps/function_output_manifest.py +++ b/openhcs/core/steps/function_output_manifest.py @@ -16,7 +16,7 @@ from openhcs.core.artifacts.plans import ArtifactOutputPlan from openhcs.core.artifacts.kinds import ArtifactType, ImageArtifactType from openhcs.core.aligned_image_payload import AlignedImageSliceContext -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.path_pattern_matching import PathPatternTemplateMatcher from openhcs.core.runtime_array_values import RuntimeArrayData from openhcs.core.runtime_image_values import ImagePayloadMetadata @@ -613,12 +613,12 @@ def relative_output_path(output_path: str, output_dir: Path) -> str: _STEP_OUTPUT_MANIFESTS: WeakKeyDictionary[ - ProcessingContext, + ExecutingContext, StepOutputManifestStore, ] = WeakKeyDictionary() -def step_output_manifest(context: ProcessingContext) -> StepOutputManifestStore: +def step_output_manifest(context: ExecutingContext) -> StepOutputManifestStore: """Return execution-local main-flow output lineage for a context.""" if context in _STEP_OUTPUT_MANIFESTS: return _STEP_OUTPUT_MANIFESTS[context] diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py index 5a0a891d2b..961903b0ea 100644 --- a/openhcs/core/steps/function_outputs.py +++ b/openhcs/core/steps/function_outputs.py @@ -27,7 +27,7 @@ from openhcs.core.compiled_step_plan import ( CompiledStepPlan, ) -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.image_file_serialization import ( ImageFileFormat, ImageFileSourceMetadata, @@ -108,7 +108,7 @@ def stream_payload_summary(data: object) -> str: def finalize_function_step_outputs( - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> StepExecutionObservation: """Save and publish one step's actual outputs before releasing their payloads.""" @@ -183,7 +183,7 @@ class MemoryOutputWriter: @classmethod def write_if_needed( cls, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> None: if plan.write_backend == Backend.MEMORY.value: @@ -251,7 +251,7 @@ class MaterializedImageOutputWriter: @staticmethod def write_if_needed( - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> None: materialized_output = plan.materialized_output @@ -549,7 +549,7 @@ class StreamOutputsAuthority: @staticmethod def stream_outputs( - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> None: for config_instance in plan.streaming_configs.values(): @@ -675,7 +675,7 @@ def for_plan(cls, plan: CompiledStepPlan) -> tuple[OpenHCSMetadataTarget, ...]: @classmethod def from_execution( - cls, context: ProcessingContext, plan: CompiledStepPlan + cls, context: ExecutingContext, plan: CompiledStepPlan ) -> OpenHCSMetadataTarget | None: """Resolve the target participating in this step's publication.""" return cls.from_plan(plan) @@ -683,7 +683,7 @@ def from_execution( @classmethod def for_execution( cls, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, *, artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = (), @@ -700,7 +700,7 @@ def for_execution( def production_targets( self, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> tuple[OpenHCSMetadataTarget, ...]: """Project this declaration's exact storage destinations for the step.""" @@ -708,7 +708,7 @@ def production_targets( def reconciliation_targets( self, - context: ProcessingContext, + context: ExecutingContext, *, document: OpenHCSMetadataSubdirectories | None = None, accepted_entries: Mapping[str, VirtualWorkspaceSourceProjectionEntries] | None = None, @@ -717,14 +717,14 @@ def reconciliation_targets( return (self,) def produced_records( - self, context: ProcessingContext, plan: CompiledStepPlan + self, context: ExecutingContext, plan: CompiledStepPlan ) -> tuple[ProducedOutputSemantics, ...]: """Targets without main-flow image storage have no produced records.""" return () def runtime_artifact_projection_paths( self, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, *, produced_projections: Mapping[ @@ -739,13 +739,13 @@ def runtime_artifact_projection_paths( ) return () - def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: + def stored_output_paths(self, context: ExecutingContext) -> tuple[str, ...]: """Return this declaration's outputs eligible for publication.""" return tuple( context.filemanager.list_image_files(self.output_dir, self.backend) ) - def contains_outputs(self, context: ProcessingContext) -> bool: + def contains_outputs(self, context: ExecutingContext) -> bool: """Return whether this declared destination has publishable outputs.""" if context.filemanager is None: @@ -756,7 +756,7 @@ def contains_outputs(self, context: ProcessingContext) -> bool: def write( self, - context: ProcessingContext, + context: ExecutingContext, *, produced_plan: CompiledStepPlan | None = None, metadata_writer: AtomicMetadataWriter | None = None, @@ -805,7 +805,7 @@ def write( def produced_projection_entries( self, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> VirtualWorkspaceSourceProjectionEntries: """Project this step's update, including an empty non-image update. @@ -918,7 +918,7 @@ def produced_projection_entries( def project_runtime_artifacts( self, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, *, produced_projections: Mapping[ @@ -994,7 +994,7 @@ def project_runtime_artifacts( def persisted_image_metadata( self, - context: ProcessingContext, + context: ExecutingContext, *, destination: str, payload, @@ -1032,7 +1032,7 @@ def persisted_image_metadata( @classmethod def observe_for_step( cls, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, *, artifact_materializations: tuple[MaterializedRuntimeArtifact, ...] = (), @@ -1050,13 +1050,13 @@ def observe_for_step( @classmethod def finalize_completed_dataset( cls, - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, ExecutingContext], *, runtime_observations: Iterable[RuntimeExecutionObservation] = (), ) -> None: """Publish completed or partial saved outputs once at the receiving boundary.""" - target_contexts: dict[OpenHCSMetadataTarget, ProcessingContext] = {} + target_contexts: dict[OpenHCSMetadataTarget, ExecutingContext] = {} for context in compiled_contexts.values(): for plan in context.step_plans.values(): if not plan.create_openhcs_metadata: @@ -1076,7 +1076,7 @@ def finalize_completed_dataset( target_contexts.setdefault(target, context) targets_by_document: dict[ - Path, dict[OpenHCSMetadataTarget, ProcessingContext] + Path, dict[OpenHCSMetadataTarget, ExecutingContext] ] = {} for owner, context in target_contexts.items(): targets_by_document.setdefault( @@ -1097,7 +1097,7 @@ class ProducedImageMetadataCapability: """Main-flow and checkpoint targets share the typed image-record projection.""" def produced_records( - self, context: ProcessingContext, plan: CompiledStepPlan + self, context: ExecutingContext, plan: CompiledStepPlan ) -> tuple[ProducedOutputSemantics, ...]: return step_output_manifest(context).image_records_for(plan) @@ -1111,7 +1111,7 @@ class PrimaryImageMetadataTarget( @classmethod def from_execution( - cls, context: ProcessingContext, plan: CompiledStepPlan + cls, context: ExecutingContext, plan: CompiledStepPlan ) -> PrimaryImageMetadataTarget | None: if not step_output_manifest(context).image_records_for(plan): return None @@ -1170,13 +1170,13 @@ def from_plan( class RuntimeArtifactMetadataTarget(OpenHCSMetadataTarget): """Saved artifacts own result destinations independently of image storage.""" - def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: + def stored_output_paths(self, context: ExecutingContext) -> tuple[str, ...]: """Include every saved format in this declared result destination.""" return tuple(context.filemanager.list_files(self.output_dir, self.backend)) def production_targets( self, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, ) -> tuple[RuntimeArtifactMetadataTarget, ...]: """Derive directories from the same writer outputs used to save artifacts.""" @@ -1190,7 +1190,7 @@ def production_targets( def reconciliation_targets( self, - context: ProcessingContext, + context: ExecutingContext, *, document: OpenHCSMetadataSubdirectories | None = None, accepted_entries: Mapping[str, VirtualWorkspaceSourceProjectionEntries] | None = None, diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index c11d2eb8a3..951f4ad5fb 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -37,7 +37,7 @@ RuntimeFixedComponentValues, ) from openhcs.core.component_set import ComponentSet -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.debug import ( DebugCursor, DebugEvent, @@ -176,7 +176,7 @@ ) RuntimePayload = FunctionOutputContextualizedValue RuntimeFunctionOutput = RuntimePayload | NoMainFlowOutput | tuple[RuntimePayload, ...] -RuntimeCallableArgument = JsonValue | RuntimePayload | ProcessingContext +RuntimeCallableArgument = JsonValue | RuntimePayload | ExecutingContext RuntimeCallableKwargs = Mapping[str, RuntimeCallableArgument] EMPTY_ARTIFACT_PLANS: ArtifactOutputPlans = MappingProxyType({}) @@ -185,7 +185,7 @@ class PatternGroupExecutionScope: """Shared pattern-group execution coordinates.""" - context: ProcessingContext + context: ExecutingContext execution_plan: CompiledStepPlan compiled_group: CompiledFunctionGroup component_value: RuntimeComponentValue = None @@ -1526,7 +1526,7 @@ def execute_chain(self) -> RuntimeArrayData | NoMainFlowOutput: def _save_artifact_value( - context: ProcessingContext, + context: ExecutingContext, output_plan: ArtifactOutputPlan, value: RuntimePayload, source_payload: RuntimePayload, diff --git a/openhcs/core/steps/function_step.py b/openhcs/core/steps/function_step.py index 341e3aaf30..bfc8ccbd27 100644 --- a/openhcs/core/steps/function_step.py +++ b/openhcs/core/steps/function_step.py @@ -13,7 +13,7 @@ from openhcs.core.steps.function_execution import FunctionStepExecutor if TYPE_CHECKING: - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext FunctionEntry: TypeAlias = Callable | tuple[Callable, dict] @@ -40,7 +40,7 @@ def __init__( def process( self, - context: "ProcessingContext", + context: "ExecutingContext", step_index: int, ) -> StepExecutionObservation: return FunctionStepExecutor.execute(context, step_index) diff --git a/openhcs/core/steps/stream_component_semantics.py b/openhcs/core/steps/stream_component_semantics.py index 8df6ace26f..6aadcc3fc3 100644 --- a/openhcs/core/steps/stream_component_semantics.py +++ b/openhcs/core/steps/stream_component_semantics.py @@ -29,7 +29,7 @@ ) from openhcs.runtime.viewer_display import DeclaredAxes, ViewerSlotFamily -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxisValueProjection from openhcs.core.runtime_slice_projection import ( @@ -587,7 +587,7 @@ class StreamMetadataRootAuthority: candidate_roots: tuple[Path, ...] @classmethod - def from_context(cls, context: ProcessingContext) -> "StreamMetadataRootAuthority": + def from_context(cls, context: ExecutingContext) -> "StreamMetadataRootAuthority": roots = [] for value in ( context.dataset_root, @@ -649,7 +649,7 @@ class StreamComponentDomainProvider(ABC, metaclass=AutoRegisterMeta): def for_component( cls, *, - context: ProcessingContext, + context: ExecutingContext, component: str, metadata_roots: tuple[StreamMetadataRoot, ...], ) -> "StreamComponentDomainProvider": @@ -668,7 +668,7 @@ def for_component( def build_for_component( cls, *, - context: ProcessingContext, + context: ExecutingContext, component: str, metadata_roots: tuple[StreamMetadataRoot, ...], ) -> "StreamComponentDomainProvider": @@ -706,13 +706,13 @@ class StreamExecutionAxisDomainProvider(StreamComponentDomainProvider): def build_for_component( cls, *, - context: ProcessingContext, + context: ExecutingContext, component: str, metadata_roots: tuple[StreamMetadataRoot, ...], ) -> "StreamExecutionAxisDomainProvider": if context.execution_runtime is None: raise RuntimeError( - "Streaming component domain requires ProcessingContext.execution_runtime " + "Streaming component domain requires ExecutingContext.execution_runtime " "for the multiprocessing axis." ) values = context.execution_runtime.execution_axis_values @@ -763,7 +763,7 @@ class StreamComponentDomainProviders: def from_layout( cls, *, - context: ProcessingContext, + context: ExecutingContext, layout: ViewerComponentLayout, metadata_roots: tuple[StreamMetadataRoot, ...], ) -> "StreamComponentDomainProviders": @@ -804,7 +804,7 @@ def from_context( cls, viewer_surface: StreamingViewerSurface, *, - context: ProcessingContext, + context: ExecutingContext, source_metadata_items: StreamSourceComponentMetadataItems, ) -> "StreamComponentMessageExtraAuthority": viewer_surface = cls._scope_to_complete_components( diff --git a/openhcs/core/streaming_config_factory.py b/openhcs/core/streaming_config_factory.py index 7a53b5b21a..5d7f36c304 100644 --- a/openhcs/core/streaming_config_factory.py +++ b/openhcs/core/streaming_config_factory.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from openhcs.core.config import StreamingConfig - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext @dataclass(frozen=True, slots=True) @@ -216,10 +216,10 @@ def viewer_surface( def streaming_viewer_surface( self, - context: "ProcessingContext | None", + context: "ExecutingContext | None", ) -> StreamingViewerSurface: if context is None: - raise ValueError("Streaming viewer surface requires a ProcessingContext.") + raise ValueError("Streaming viewer surface requires a ExecutingContext.") source = ViewerStreamSourceIdentity( microscope_handler=context.microscope_handler, plate_path=context.dataset_root, diff --git a/openhcs/core/virtual_workspace_metadata.py b/openhcs/core/virtual_workspace_metadata.py index 9022081c2d..78172fd244 100644 --- a/openhcs/core/virtual_workspace_metadata.py +++ b/openhcs/core/virtual_workspace_metadata.py @@ -37,7 +37,7 @@ from openhcs.core.axes import Axis, AxisFamily, ColourAxis if TYPE_CHECKING: - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.steps.function_outputs import OpenHCSMetadataTarget @@ -217,7 +217,7 @@ def update(data): def reconcile_completed_dataset( self, metadata_path: str | Path, - target_contexts: Mapping[OpenHCSMetadataTarget, ProcessingContext], + target_contexts: Mapping[OpenHCSMetadataTarget, ExecutingContext], *, produced_entries_by_target: Mapping[ OpenHCSMetadataTarget, VirtualWorkspaceSourceProjectionEntries diff --git a/openhcs/domains/microscopy/analysis_consolidation.py b/openhcs/domains/microscopy/analysis_consolidation.py index 4d4a5c4e0a..aeb06312bb 100644 --- a/openhcs/domains/microscopy/analysis_consolidation.py +++ b/openhcs/domains/microscopy/analysis_consolidation.py @@ -11,7 +11,7 @@ import pandas as pd from openhcs.core.axes import AxisFamily -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.post_execute import PostExecuteHook from openhcs.domains.microscopy.config import ( AnalysisConsolidationConfig, @@ -93,7 +93,7 @@ def from_saved_outputs( def from_reused_outputs( cls, config: AnalysisConsolidationConfig, - context: ProcessingContext, + context: ExecutingContext, plan: CompiledStepPlan, materialization: RuntimeArtifactMaterialization, outputs: tuple[Output, ...], @@ -273,7 +273,7 @@ def combine(cls, observations): def run( self, - compiled_contexts: Mapping[str, ProcessingContext], + compiled_contexts: Mapping[str, ExecutingContext], observation: RuntimeAnalysisConsolidationInputs | None, ) -> None: if not self.analysis_consolidation_config.enabled: diff --git a/openhcs/interop/cellprofiler/analyst_export.py b/openhcs/interop/cellprofiler/analyst_export.py index 8af6cdd4a6..d38db1fcaa 100644 --- a/openhcs/interop/cellprofiler/analyst_export.py +++ b/openhcs/interop/cellprofiler/analyst_export.py @@ -35,7 +35,7 @@ RelationshipsArtifactType, ) from openhcs.core.component_group_scope import RuntimeExecutionAxisScope -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.image_file_serialization import ImageFileFormat, ImageFileRevision from openhcs.core.measurement_row_materialization import ( WideMeasurementRowAccumulator, @@ -675,7 +675,7 @@ class CPATableRowProjection: dialect: CellProfilerDatabaseColumnDialect image_set_numbering: CellProfilerImageSetNumbering - context: ProcessingContext | None = None + context: ExecutingContext | None = None def measurement_projections( self, @@ -1355,7 +1355,7 @@ class CellProfilerAnalystProjectionBuilder: source_binding_plan: CompiledSourceBindingPlan dialect: CellProfilerDatabaseColumnDialect | None = None - context: ProcessingContext | None = None + context: ExecutingContext | None = None def __post_init__(self) -> None: if not isinstance(self.source_binding_plan, CompiledSourceBindingPlan): diff --git a/openhcs/interop/cellprofiler/image_set_numbering.py b/openhcs/interop/cellprofiler/image_set_numbering.py index dd78bcb137..d78fa8ff09 100644 --- a/openhcs/interop/cellprofiler/image_set_numbering.py +++ b/openhcs/interop/cellprofiler/image_set_numbering.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.context.processing_context import ExecutingContext from dataclasses import dataclass, field from openhcs.core.component_group_scope import RuntimeExecutionAxisScope @@ -50,7 +50,7 @@ class CellProfilerImageSetNumbering: def observe_export_paths( self, - context: "ProcessingContext", + context: "ExecutingContext", paths: Sequence[str], ) -> None: """Bind actual exporter numbering to its relative bundle paths.""" diff --git a/openhcs/processing/backends/cellprofiler/export_to_database.py b/openhcs/processing/backends/cellprofiler/export_to_database.py index 3dd9458538..1ddeacf0b2 100644 --- a/openhcs/processing/backends/cellprofiler/export_to_database.py +++ b/openhcs/processing/backends/cellprofiler/export_to_database.py @@ -16,7 +16,7 @@ SpecialArtifactType, ) from openhcs.core.callable_contract import FunctionStepExecutionScope -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.pipeline.function_contracts import ( execution_scope, runtime_bound_parameters, @@ -833,7 +833,7 @@ def _group_setting_records( @staticmethod def prepare_bundle( artifact_batch: RuntimeArtifactBatch, - context: ProcessingContext | None, + context: ExecutingContext | None, parameters: Mapping[str, Any], *, map_partition_invocations: ( @@ -920,7 +920,7 @@ def _partitioned_database_export( def export_to_database( *, artifact_batch: RuntimeArtifactBatch, - context: ProcessingContext, + context: ExecutingContext, sqlite_file: str = "DefaultDB.db", experiment_name: str = "MyExpt", add_table_prefix: bool = False, diff --git a/openhcs/processing/backends/cellprofiler/save_images.py b/openhcs/processing/backends/cellprofiler/save_images.py index 2285f7359a..a58060b218 100644 --- a/openhcs/processing/backends/cellprofiler/save_images.py +++ b/openhcs/processing/backends/cellprofiler/save_images.py @@ -20,7 +20,7 @@ MaterializationSourceIdentityRelation, ) from openhcs.core.artifacts.kinds import ImageArtifactType, MeasurementsArtifactType -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.image_file_serialization import ImagePayloadUint8Strategy from openhcs.core.memory import numpy from openhcs.core.measurement_row_materialization import ( @@ -920,7 +920,7 @@ def _recorded_save_images_rows( filename_suffix: str, file_format: SaveImagesFileFormat, output_location: str | None, - context: ProcessingContext | None, + context: ExecutingContext | None, ) -> DataclassMeasurementColumnarRows: suffix = filename_suffix if append_suffix else "" if filename_method is SaveImagesFilenameMethod.FROM_IMAGE_FILENAME: @@ -1044,7 +1044,7 @@ def save_images_with_measurements( series_axis: SaveImagesSeriesAxis = SaveImagesSeriesAxis.TIME, lossless_compression: bool = True, slice_index: int = 0, - context: ProcessingContext | None = None, + context: ExecutingContext | None = None, ) -> tuple[RuntimeArrayData, RuntimeArrayData, DataclassMeasurementColumnarRows]: """Prepare an image for saving and record its output-file measurements. diff --git a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py index b3b917548c..3104cf6494 100644 --- a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py +++ b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py @@ -31,7 +31,7 @@ SpecialArtifactType, ) from openhcs.core.callable_contract import FunctionStepExecutionScope -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.measurement_row_materialization import ( MeasurementSparseColumnarRows, MEASUREMENT_SPARSE_CELL, @@ -597,7 +597,7 @@ def prepare_spreadsheet_bundle( ), add_filename_prefix: bool = True, filename_prefix: str = "MyExpt_", - context: ProcessingContext | None = None, + context: ExecutingContext | None = None, image_numbers: CellProfilerImageSetNumbering | None = None, experiment_tables: tuple[MeasurementTable, ...] | None = None, ) -> dict[str, ColumnarCsvOutput]: @@ -1560,7 +1560,7 @@ def _partitioned_spreadsheet_export( """Use existing lanes only where exact source domains separate the export.""" batch = request.artifact_batch kwargs = {**runtime_callable_defaults(request.func), **request.kwargs} - kwargs.pop(ProcessingContext.require_parameter_name(), None) + kwargs.pop(ExecutingContext.require_parameter_name(), None) path_templates = ( str(kwargs["output_directory"]), str(kwargs["filename_prefix"]) if kwargs["add_filename_prefix"] else "", @@ -1825,7 +1825,7 @@ def export_to_spreadsheet( add_filename_prefix: bool = True, filename_prefix: str = "MyExpt_", artifact_batch: RuntimeArtifactBatch, - context: ProcessingContext | None = None, + context: ExecutingContext | None = None, ) -> dict[str, ColumnarCsvOutput]: """Render one plate's exact contract-selected spreadsheet file bundle. diff --git a/openhcs/runtime/zmq_execution_observation.py b/openhcs/runtime/zmq_execution_observation.py index 04349bf56e..b02a1b3081 100644 --- a/openhcs/runtime/zmq_execution_observation.py +++ b/openhcs/runtime/zmq_execution_observation.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from pathlib import Path -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ExecutingContext from openhcs.core.orchestrator.execution_result import ( ExecutionResult, RuntimeExecutionObservation, @@ -153,7 +153,7 @@ class ZMQRuntimeExecutionObservationExport: def from_execution( cls, *, - compiled_contexts: Mapping[str, ProcessingContext], + executed_contexts: Mapping[str, ExecutingContext], execution_results: Mapping[str, ExecutionResult], output_roots: tuple[Path, ...], runtime_observations: tuple[RuntimeExecutionObservation, ...], @@ -161,12 +161,12 @@ def from_execution( execution_id: str | None = None, ) -> ZMQRuntimeExecutionObservationExport: observation = RuntimeArtifactExecutionObservation.from_contexts( - compiled_contexts, runtime_observations=runtime_observations + executed_contexts, runtime_observations=runtime_observations ) return cls( schema_version=ZMQ_RUNTIME_OBSERVATION_EXPORT_SCHEMA_VERSION, - expectation=RuntimeArtifactExecutionExpectation.from_compiled_contexts( - compiled_contexts + expectation=RuntimeArtifactExecutionExpectation.from_executed_contexts( + executed_contexts ), records_by_axis=dict(observation.records_by_axis), exports=observation.exports, diff --git a/openhcs/runtime/zmq_execution_server.py b/openhcs/runtime/zmq_execution_server.py index 5f71b1e779..0238e4a802 100644 --- a/openhcs/runtime/zmq_execution_server.py +++ b/openhcs/runtime/zmq_execution_server.py @@ -846,7 +846,7 @@ def _export_runtime_observation( ) else: export = ZMQRuntimeExecutionObservationExport.from_execution( - compiled_contexts=execution_bundle.runtime_contexts, + executed_contexts=execution_results.executed_contexts, execution_results=execution_results, runtime_observations=execution_results.runtime_observations, output_roots=output_roots, diff --git a/tests/integration/test_neurite_outgrowth_synthetic_plate.py b/tests/integration/test_neurite_outgrowth_synthetic_plate.py index 89081a20ef..ce18c71993 100644 --- a/tests/integration/test_neurite_outgrowth_synthetic_plate.py +++ b/tests/integration/test_neurite_outgrowth_synthetic_plate.py @@ -192,7 +192,7 @@ def observe_dense_output(output, components): for output in compiled_plan.artifact_outputs.values() if output.name == "neurite_candidate_mask" ) - assert checkpoint_plan.group_component is Microscopy.Site + assert checkpoint_plan.group_scope.component is Microscopy.Site assert checkpoint_plan.group_scope_sources() assert checkpoint_plan.source_context_source() is None assert compiled_plan.variable_components == [Microscopy.Channel] diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index ef8cc159d3..db73fc391b 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -766,7 +766,7 @@ def _artifact_input_edge_for_test( spec=spec, invocation_scope=scope, producer_selection_scope=UngroupedScope(), - component_scopes=() if scope.is_ungrouped else (scope,), + component_scopes=() if not scope.as_grouped() else (scope,), consumer_variable_components=(), ), spec=spec, @@ -4304,7 +4304,7 @@ def _storage_seed_plans( name=storage_plan.name, path=storage_plan.path_for_runtime_query(group_key), artifact_type=storage_plan.artifact_type, - group_scope=ComponentGroupScope.of((group_key,), component=storage_plan.group_component), + group_scope=ComponentGroupScope.of((group_key,), component=storage_plan.group_scope.component), variable_components=storage_plan.variable_components, component_domains=storage_plan.component_domains, paths_by_group={ @@ -4312,7 +4312,7 @@ def _storage_seed_plans( }, sidecar_role=storage_plan.sidecar_role, ) - for group_key in (storage_plan.group_keys or (None,)) + for group_key in (storage_plan.group_scope.keys or (None,)) ) @staticmethod diff --git a/tests/unit/test_cellprofiler_runtime_adapter.py b/tests/unit/test_cellprofiler_runtime_adapter.py index 3b2aa24012..eed6279606 100644 --- a/tests/unit/test_cellprofiler_runtime_adapter.py +++ b/tests/unit/test_cellprofiler_runtime_adapter.py @@ -1397,7 +1397,7 @@ def test_cellprofiler_adapter_does_not_select_relationship_from_current_source_p name=relationship_name, path=output_plan.path, artifact_type=RelationshipsArtifactType, - group_scope=ComponentGroupScope.of(output_plan.group_keys, component=output_plan.group_component), + group_scope=ComponentGroupScope.of(output_plan.group_scope.keys, component=output_plan.group_scope.component), paths_by_group=output_plan.paths_by_group, ), invocation_scope=UngroupedScope(), @@ -1495,7 +1495,7 @@ def test_cellprofiler_adapter_aligns_grouped_relationships_to_runtime_slices(): name=relationship_name, path=output_plan.path, artifact_type=RelationshipsArtifactType, - group_scope=ComponentGroupScope.of(output_plan.group_keys, component=output_plan.group_component), + group_scope=ComponentGroupScope.of(output_plan.group_scope.keys, component=output_plan.group_scope.component), paths_by_group=output_plan.paths_by_group, ), invocation_scope=UngroupedScope(), @@ -4689,21 +4689,21 @@ def test_declared_measurement_inputs_require_producer_declared_slice_indexes(): name=MEASUREMENTS, path=output_plan.path, artifact_type=MeasurementsArtifactType, - group_scope=ComponentGroupScope.of(output_plan.group_keys, component=Microscopy.Site), + group_scope=ComponentGroupScope.of(output_plan.group_scope.keys, component=Microscopy.Site), paths_by_group=output_plan.paths_by_group, ), invocation_scope=ComponentGroupScope.of( ("1",), component=Microscopy.Channel ), producer_selection_scope=ComponentGroupScope.of( - output_plan.group_keys, component=Microscopy.Site + output_plan.group_scope.keys, component=Microscopy.Site ), component_scopes=( ComponentGroupScope.of( ("1",), component=Microscopy.Channel ), ComponentGroupScope.of( - output_plan.group_keys, component=Microscopy.Site + output_plan.group_scope.keys, component=Microscopy.Site ), ), consumer_variable_components=(Microscopy.Site,), diff --git a/tests/unit/test_debug_runtime.py b/tests/unit/test_debug_runtime.py index a1c04c1d7a..2db4396218 100644 --- a/tests/unit/test_debug_runtime.py +++ b/tests/unit/test_debug_runtime.py @@ -314,7 +314,7 @@ def compiled_invocation( producer_selection_scope=plan.group_scope, component_scopes=( () - if plan.group_scope.is_ungrouped + if not plan.group_scope.as_grouped() else (plan.group_scope,) ), ), diff --git a/tests/unit/test_funcstep_contract_validator.py b/tests/unit/test_funcstep_contract_validator.py index 1449833d64..d39125b0ee 100644 --- a/tests/unit/test_funcstep_contract_validator.py +++ b/tests/unit/test_funcstep_contract_validator.py @@ -144,7 +144,7 @@ def _compiled_pattern_with_exact_input_edges( invocation_scope=invocation_scope, producer_selection_scope=producer_scope, component_scopes=( - () if producer_scope.is_ungrouped else (producer_scope,) + () if not producer_scope.as_grouped() else (producer_scope,) ), ), ) diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index 5350f15cf7..eaedaa016f 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -196,7 +196,7 @@ class CoreExecutionRequest: def _execute_function_core(request: CoreExecutionRequest): pattern = ( request.func_callable - if request.execution_group_scope.is_ungrouped + if not request.execution_group_scope.as_grouped() else {request.group_key: [request.func_callable]} ) compiled_pattern = compile_function_pattern( @@ -241,7 +241,7 @@ def _execute_function_core(request: CoreExecutionRequest): artifact_outputs=request.artifact_outputs, ) component_value = ( - None if request.execution_group_scope.is_ungrouped else request.group_key + None if not request.execution_group_scope.as_grouped() else request.group_key ) runtime_scope = PatternGroupData( matching_files=[ diff --git a/tests/unit/test_k7_execution_protocol_guards.py b/tests/unit/test_k7_execution_protocol_guards.py new file mode 100644 index 0000000000..335c1b82f6 --- /dev/null +++ b/tests/unit/test_k7_execution_protocol_guards.py @@ -0,0 +1,195 @@ +"""K7 guards: scope variants, context phases and execution results are types.""" + +from __future__ import annotations + +import ast +import dataclasses +import inspect +import pickle +import sys +from pathlib import Path + +import pytest + +import openhcs +from openhcs.core.artifacts.plans import ArtifactPlan +from openhcs.core.component_group_scope import ( + ComponentGroupScope, + DynamicGroupScope, + SingleKeyScope, + StaticGroupScope, + UngroupedScope, +) +import openhcs.core.context.processing_context as processing_context +from openhcs.core.context.processing_context import ( + CompiledContext, + CompilingContext, + ExecutingContext, + ProcessingContext, +) +from openhcs.core.orchestrator import execution_result +from openhcs.core.orchestrator.execution_result import ( + ExecutionCancelled, + ExecutionFailed, + ExecutionResult, + ExecutionSucceeded, + MergeAllRuntimeRecords, + MergeDatasetInputRecords, + OmitRuntimeRecords, + RuntimeObservationMode, +) +from openhcs.domains.microscopy.axes import Microscopy + +REPO_ROOT = Path(openhcs.__file__).resolve().parents[1] + +# Names whose only meaning was a None/flag variant re-decided by the caller. +DELETED_NAMES = frozenset( + { + "is_ungrouped", + "has_single_static_key", + "require_single_static_key", + "PathPlannerGroupScope", + "producer_group_scope", + "has_dynamic_group_scope", + "single_group_key", + "is_frozen", + "_is_frozen", + "bind_execution_runtime", + "reset_completed_step_outputs", + "ZMQExecutionAxisOutcome", + "is_cancelled", + } +) + + +def _concrete_subclasses(base: type) -> set[type]: + """Leaf members of a family, as their modules publish them.""" + found: set[type] = set() + pending = list(base.__subclasses__()) + while pending: + cls = pending.pop() + pending.extend(cls.__subclasses__()) + module = sys.modules.get(cls.__module__) + if ( + cls.__module__.startswith("openhcs.") + and getattr(module, cls.__name__, None) is cls + and not inspect.isabstract(cls) + and not any( + getattr(module, sub.__name__, None) is sub + for sub in cls.__subclasses__() + ) + ): + found.add(cls) + return found + + +def _names(tree: ast.AST) -> set[str]: + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Name): + names.add(node.id) + elif isinstance(node, ast.Attribute): + names.add(node.attr) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, ast.arg): + names.add(node.arg) + elif isinstance(node, ast.keyword) and node.arg: + names.add(node.arg) + return names + + +def test_replaced_execution_protocol_names_do_not_exist() -> None: + offenders = { + path.relative_to(REPO_ROOT).as_posix(): sorted( + DELETED_NAMES & _names(ast.parse(path.read_text())) + ) + for path in (REPO_ROOT / "openhcs").rglob("*.py") + } + assert {path: names for path, names in offenders.items() if names} == {} + + +def test_scope_variants_are_the_declared_family() -> None: + assert _concrete_subclasses(ComponentGroupScope) == { + UngroupedScope, + DynamicGroupScope, + StaticGroupScope, + SingleKeyScope, + } + assert not hasattr(ComponentGroupScope, "from_raw") + + +def test_artifact_plans_hold_one_group_scope() -> None: + names = {field.name for field in dataclasses.fields(ArtifactPlan)} + assert "group_scope" in names + assert not names & {"group_keys", "group_component"} + + +def test_context_phases_and_results_are_the_declared_families() -> None: + assert _concrete_subclasses(ProcessingContext) == { + CompilingContext, + CompiledContext, + ExecutingContext, + } + assert _concrete_subclasses(ExecutionResult) == { + ExecutionSucceeded, + ExecutionFailed, + ExecutionCancelled, + } + assert set(RuntimeObservationMode.__subclasses__()) == { + OmitRuntimeRecords, + MergeAllRuntimeRecords, + MergeDatasetInputRecords, + } + assert not hasattr(execution_result, "ExecutionStatus") + assert not hasattr(processing_context, "_is_frozen") + + +@pytest.mark.parametrize( + ("scope", "keys", "contained", "outside"), + [ + (UngroupedScope(), (None,), None, "1"), + (DynamicGroupScope(Microscopy.Channel), (None,), "7", None), + (StaticGroupScope(Microscopy.Channel, ("1", "2")), ("1", "2"), "2", "3"), + (SingleKeyScope(Microscopy.Channel, "1"), ("1",), "1", "2"), + ], +) +def test_each_scope_variant_reads_back_from_its_raw_spelling( + scope, keys, contained, outside +) -> None: + assert ComponentGroupScope.of(keys, component=scope.component) == scope + assert scope.contains_runtime_key(contained) + assert not scope.contains_runtime_key(outside) + assert pickle.loads(pickle.dumps(scope)) == scope + + +def test_context_phases_carry_only_their_own_state(tmp_path) -> None: + compiling = CompilingContext(axis_id="A01", filemanager=object()) + assert not hasattr(compiling, "runtime_value_store") + compiled = compiling.freeze() + assert not hasattr(compiled, "runtime_value_store") + with pytest.raises(AttributeError): + compiled.axis_id = "B01" + first = compiled.executing() + second = compiled.executing() + assert first.runtime_value_store is not second.runtime_value_store + assert first.axis_id == compiled.axis_id + with pytest.raises(AttributeError): + first.step_plans = {} + restored = pickle.loads(pickle.dumps(compiled)) + assert restored.axis_id == "A01" + with pytest.raises(AttributeError): + restored.axis_id = "B01" + + +def test_stopped_results_own_their_message_and_combination() -> None: + failed = ExecutionFailed("A01", error_message="boom").at_combination( + "A01__combo_0", execution_result.RuntimeExecutionObservation() + ) + assert not failed.is_success() + assert failed.failed_combination == "A01__combo_0" + assert ExecutionSucceeded("A01").is_success() + assert ExecutionCancelled("A01", error_message="stop").stops_lane + assert not {"error_message", "failed_combination"} & { + field.name for field in dataclasses.fields(ExecutionSucceeded) + } diff --git a/tests/unit/test_path_planner_materialization.py b/tests/unit/test_path_planner_materialization.py index 39a2d88d93..d91dffd4e7 100644 --- a/tests/unit/test_path_planner_materialization.py +++ b/tests/unit/test_path_planner_materialization.py @@ -1612,7 +1612,7 @@ def identify(main_image, Stain1): ) assert maps.group_scope.keys == expected_channel_scope.keys assert maps.group_scope.component is expected_channel_scope.component - assert maps.outputs[object_output.ref()].group_keys == ("1",) + assert maps.outputs[object_output.ref()].group_scope.keys == ("1",) assert maps.outputs[object_output.ref()].variable_components == ( Microscopy.Site, ) @@ -1970,10 +1970,10 @@ def test_declared_group_lineage_scopes_outputs_without_rewriting_execution(): "FilterObjects_8_measurements", MeasurementsArtifactType, ).ref() - assert maps.outputs[filtered_tiles_ref].group_keys == ("1",) - assert maps.outputs[measurements_ref].group_keys == ("1",) - assert maps.outputs[filtered_tiles_ref].group_component is Microscopy.Channel - assert maps.outputs[measurements_ref].group_component is Microscopy.Channel + assert maps.outputs[filtered_tiles_ref].group_scope.keys == ("1",) + assert maps.outputs[measurements_ref].group_scope.keys == ("1",) + assert maps.outputs[filtered_tiles_ref].group_scope.component is Microscopy.Channel + assert maps.outputs[measurements_ref].group_scope.component is Microscopy.Channel assert maps.group_scope == ComponentGroupScope.of( ("2",), component=Microscopy.Channel, @@ -2019,11 +2019,11 @@ def test_declared_group_lineage_uses_main_flow_scope_without_artifact_plan(): "MeasureColocalization_4_measurements", MeasurementsArtifactType, ).ref() - assert maps.outputs[measurements_ref].group_keys == ( + assert maps.outputs[measurements_ref].group_scope.keys == ( "1", "2", ) - assert maps.outputs[measurements_ref].group_component is Microscopy.Site + assert maps.outputs[measurements_ref].group_scope.component is Microscopy.Site def test_prior_main_flow_artifact_scopes_output_without_rewriting_execution(): @@ -2073,7 +2073,7 @@ def test_prior_main_flow_artifact_scopes_output_without_rewriting_execution(): component=Microscopy.Channel, ) nuclei_ref = ArtifactSpec.output("Nuclei", ObjectLabelsArtifactType).ref() - assert maps.outputs[nuclei_ref].group_keys == ("3",) + assert maps.outputs[nuclei_ref].group_scope.keys == ("3",) def test_dict_invocation_lineage_uses_its_non_plan_input_group_scope(): @@ -2110,7 +2110,7 @@ def test_dict_invocation_lineage_uses_its_non_plan_input_group_scope(): ) objects_ref = ArtifactSpec.output("Objects1", ObjectLabelsArtifactType).ref() - assert maps.outputs[objects_ref].group_keys == ("1",) + assert maps.outputs[objects_ref].group_scope.keys == ("1",) def test_measurement_output_scope_compiles_exact_cross_group_consumer_edge(): @@ -2187,7 +2187,7 @@ def test_measurement_output_scope_compiles_exact_cross_group_consumer_edge(): measurement_name, MeasurementsArtifactType, ).ref() - assert maps.outputs[measurement_ref].group_keys == ("2",) + assert maps.outputs[measurement_ref].group_scope.keys == ("2",) _record_declared_output(planner, maps.outputs[measurement_ref]) measurement_input = ArtifactSpec.input( measurement_name, @@ -2364,8 +2364,8 @@ def test_artifact_managed_single_source_output_retains_source_group_scope(): assert maps.group_scope == UngroupedScope() nuclei_ref = ArtifactSpec.output("Nuclei", ObjectLabelsArtifactType).ref() - assert maps.outputs[nuclei_ref].group_keys == ("1",) - assert maps.outputs[nuclei_ref].group_component is Microscopy.Channel + assert maps.outputs[nuclei_ref].group_scope.keys == ("1",) + assert maps.outputs[nuclei_ref].group_scope.component is Microscopy.Channel def test_declared_group_lineage_unions_compatible_source_groups(): @@ -2433,7 +2433,7 @@ def test_declared_group_lineage_unions_compatible_source_groups(): "Measurements", MeasurementsArtifactType, ).ref() - assert maps.outputs[measurements_ref].group_keys == ("1", "2") + assert maps.outputs[measurements_ref].group_scope.keys == ("1", "2") assert maps.relation_source_scopes[first_ref] == ( ComponentGroupScope.of(("1",), component=Microscopy.Channel) ) @@ -2518,8 +2518,8 @@ def test_collected_lineage_outputs_use_relation_owned_group_scope(): "Measurements", MeasurementsArtifactType, ).ref() - assert maps.outputs[measurements_ref].group_keys == (None,) - assert maps.outputs[measurements_ref].group_component is Microscopy.Site + assert maps.outputs[measurements_ref].group_scope.keys == (None,) + assert maps.outputs[measurements_ref].group_scope.component is Microscopy.Site def test_output_lineage_uses_input_qualified_consumer_scope(): @@ -2591,7 +2591,7 @@ def test_output_lineage_uses_input_qualified_consumer_scope(): component=Microscopy.Channel, ) ) - assert output_plan.group_keys == ("1",) + assert output_plan.group_scope.keys == ("1",) assert output_plan.group_scope_sources_by_group == { "1": (prior_measurements.ref(),), } @@ -2753,7 +2753,7 @@ def test_real_object_measurement_preserves_selected_labels_group_scope( available_artifact_producers=( ArtifactProducer( ArtifactSpec.output("Cells", ObjectLabelsArtifactType), - groups=labels_plan.group_keys, + groups=labels_plan.group_scope.keys, invocation_keys=(), producer_step_index=2, ), @@ -2813,8 +2813,8 @@ def test_real_object_measurement_preserves_selected_labels_group_scope( maps = planner.artifacts.compile_plan_maps(snapshot, 3, declarations, group_scope) measurement_plan = maps.outputs[measurement.ref()] - assert maps.inputs[labels_input.ref()].group_keys == labels_plan.group_keys - assert measurement_plan.group_keys == ("2",) + assert maps.inputs[labels_input.ref()].group_scope.keys == labels_plan.group_scope.keys + assert measurement_plan.group_scope.keys == ("2",) assert measurement_plan.group_scope_sources_by_group == { "2": (labels_input.ref(),), } @@ -2904,7 +2904,7 @@ def test_declared_group_lineage_cannot_rewrite_scalar_step_execution_scope(): component=Microscopy.Channel, ) masked_ref = ArtifactSpec.output("MembMasked", ImageArtifactType).ref() - assert maps.outputs[masked_ref].group_keys == ("3",) + assert maps.outputs[masked_ref].group_scope.keys == ("3",) assert maps.outputs[masked_ref].relations == ( GroupLineageSourceRelation( ArtifactSpec.input( @@ -2919,7 +2919,7 @@ def test_declared_group_lineage_cannot_rewrite_scalar_step_execution_scope(): monolayer_ref, ) assert maps.inputs[monolayer_ref].path == ("/memory/MonolayerMask_1.pkl") - assert maps.inputs[monolayer_ref].group_keys == ("1",) + assert maps.inputs[monolayer_ref].group_scope.keys == ("1",) def test_artifact_output_storage_scope_is_independent_of_execution_scope(): @@ -2991,7 +2991,7 @@ def test_artifact_output_storage_scope_is_independent_of_execution_scope(): component=Microscopy.Channel, ) ratio_ref = ArtifactSpec.output("Ratio", MeasurementsArtifactType).ref() - assert maps.outputs[ratio_ref].group_keys == ("1",) + assert maps.outputs[ratio_ref].group_scope.keys == ("1",) assert tuple(maps.inputs) == ( ArtifactSpec.input("Nuclei", ObjectLabelsArtifactType).ref(), ArtifactSpec.input("PriorMeasurements", MeasurementsArtifactType).ref(), @@ -3070,10 +3070,10 @@ def test_each_output_storage_scope_is_independent_of_execution_scope(): "ambiguous", MeasurementsArtifactType, ).ref() - assert ComponentGroupScope.from_output_plan(maps.outputs[scoped_ref]) == ( + assert maps.outputs[scoped_ref].group_scope == ( relation_scope ) - assert ComponentGroupScope.from_output_plan(maps.outputs[ambiguous_ref]) == ( + assert maps.outputs[ambiguous_ref].group_scope == ( execution_scope ) @@ -3177,8 +3177,8 @@ def test_source_binding_component_identity_narrows_declared_output_lineage(): ) cells_ref = ArtifactSpec.output("Cells", ObjectLabelsArtifactType).ref() - assert maps.outputs[cells_ref].group_keys == ("3",) - assert maps.outputs[cells_ref].group_component is Microscopy.Channel + assert maps.outputs[cells_ref].group_scope.keys == ("3",) + assert maps.outputs[cells_ref].group_scope.component is Microscopy.Channel def test_source_binding_identity_scopes_outputs_without_execution_fanout(): @@ -3222,8 +3222,8 @@ def test_source_binding_identity_scopes_outputs_without_execution_fanout(): assert maps.group_scope == UngroupedScope() cells_ref = ArtifactSpec.output("Cells", ObjectLabelsArtifactType).ref() - assert maps.outputs[cells_ref].group_keys == ("3",) - assert maps.outputs[cells_ref].group_component is Microscopy.Channel + assert maps.outputs[cells_ref].group_scope.keys == ("3",) + assert maps.outputs[cells_ref].group_scope.component is Microscopy.Channel def test_image_object_outputs_keep_declared_image_execution_group_scope(): @@ -3291,7 +3291,7 @@ def test_image_object_outputs_keep_declared_image_execution_group_scope(): "MeasureObjectIntensity_7_measurements", MeasurementsArtifactType, ).ref() - assert maps.outputs[measurements_ref].group_keys == ("2",) + assert maps.outputs[measurements_ref].group_scope.keys == ("2",) def test_group_lineage_source_resolves_prior_main_flow_output_without_store_input(): @@ -3346,7 +3346,7 @@ def test_group_lineage_source_resolves_prior_main_flow_output_without_store_inpu "Filtered_tiles", ObjectLabelsArtifactType, ).ref() - assert maps.outputs[filtered_tiles_ref].group_keys == ("1",) + assert maps.outputs[filtered_tiles_ref].group_scope.keys == ("1",) def test_resolved_pipeline_owns_invocation_aware_artifact_declarations(): @@ -4015,8 +4015,8 @@ def test_non_dict_group_by_namespaces_artifact_outputs_with_dynamic_component(): output_plan = maps.outputs[ ArtifactSpec.output("segmentation_masks", ObjectLabelsArtifactType).ref() ] - assert output_plan.group_keys == (None,) - assert output_plan.group_component is Microscopy.Channel + assert output_plan.group_scope.keys == (None,) + assert output_plan.group_scope.component is Microscopy.Channel def test_non_dict_group_by_uses_source_binding_identity_for_pipeline_start_scope(): @@ -4116,7 +4116,7 @@ def save_declared_payload(image): else ComponentGroupScope.of(("3",), component=Microscopy.Channel) ) assert scope == expected - assert not scope.contains_runtime_key("1") or scope.is_dynamic + assert not scope.contains_runtime_key("1") or not scope.has_compiled_keys def test_main_flow_source_anchor_restricts_execution_to_its_exact_channel(): @@ -4249,7 +4249,7 @@ def measure(image, CropBlue): component=Microscopy.Channel, ) ) - assert ComponentGroupScope.from_output_plan(maps.outputs[measurements.ref()]) == ( + assert maps.outputs[measurements.ref()].group_scope == ( execution_scope ) edge = next(compiled.iter_invocations()).artifact_input_edges[0] @@ -4783,8 +4783,8 @@ def test_artifact_input_plan_preserves_single_grouped_producer_scope(): illumination_ref = ArtifactSpec.input("illumination", ImageArtifactType).ref() plan = inputs[illumination_ref] - assert plan.group_keys == ("1",) - assert plan.group_component is Microscopy.Channel + assert plan.group_scope.keys == ("1",) + assert plan.group_scope.component is Microscopy.Channel assert plan.path == "/memory/illumination_channel_1.pkl" assert plan.paths_by_group == {"1": "/memory/illumination_channel_1.pkl"} @@ -5000,7 +5000,7 @@ def consume(main_image, relationships): consumer_variable_components=ComponentSet((Microscopy.Site,)), ) edge = next(compiled.iter_invocations()).artifact_input_edges[0] - assert edge.projection.invocation_scope.is_ungrouped + assert not edge.projection.invocation_scope.as_grouped() assert edge.projection.producer_selection_scope == ( input_plan.group_scope ) @@ -5242,8 +5242,8 @@ def test_artifact_input_plan_preserves_multi_grouped_producer_across_components( illumination_ref = ArtifactSpec.input("illumination", ImageArtifactType).ref() plan = inputs[illumination_ref] - assert plan.group_keys == ("1", "2") - assert plan.group_component is Microscopy.Channel + assert plan.group_scope.keys == ("1", "2") + assert plan.group_scope.component is Microscopy.Channel assert plan.paths_by_group == { "1": "/memory/illumination_channel_1.pkl", "2": "/memory/illumination_channel_2.pkl", diff --git a/tests/unit/test_routed_composite_execution.py b/tests/unit/test_routed_composite_execution.py index 5a31728f79..b8f11a77a1 100644 --- a/tests/unit/test_routed_composite_execution.py +++ b/tests/unit/test_routed_composite_execution.py @@ -34,6 +34,9 @@ ) from openhcs.domains.microscopy.axes import Microscopy from openhcs.microscopes.imagexpress import ImageXpressHandler +from openhcs.core.component_group_scope import ( + UngroupedScope, +) @pytest.mark.parametrize("channels", [(1, 2), (2, 3)]) @@ -140,7 +143,7 @@ def test_reduced_channel_stack_is_the_next_steps_only_input( ) context = compiled.runtime_contexts["A01"] if not reduce_channels: - assert not context.step_plans[2].execution_group_scope.is_ungrouped + assert context.step_plans[2].execution_group_scope.as_grouped() if stitch: context = compiled.runtime_contexts["A01"] position_step_index = 2 + project_z @@ -152,7 +155,7 @@ def test_reduced_channel_stack_is_the_next_steps_only_input( ) assert context.step_plans[ position_step_index - ].execution_group_scope.is_ungrouped + ].execution_group_scope == UngroupedScope() result = orchestrator.execute_compiled_bundle( execution_bundle=compiled, max_workers=1, diff --git a/tests/unit/test_runtime_execution_validation.py b/tests/unit/test_runtime_execution_validation.py index 66f20f7a9f..c60ed82f59 100644 --- a/tests/unit/test_runtime_execution_validation.py +++ b/tests/unit/test_runtime_execution_validation.py @@ -343,7 +343,7 @@ def test_compiled_artifact_viewer_expectations_preserve_full_producers() -> None backend="memory", ) - expectation = RuntimeArtifactExecutionExpectation.from_compiled_contexts( + expectation = RuntimeArtifactExecutionExpectation.from_executed_contexts( {"A01": context} ) @@ -462,7 +462,7 @@ def test_empty_roi_materialization_does_not_invent_viewer_layer() -> None: backend="memory", ) - expectation = RuntimeArtifactExecutionExpectation.from_compiled_contexts( + expectation = RuntimeArtifactExecutionExpectation.from_executed_contexts( {"A01": context} ) @@ -514,7 +514,7 @@ def test_runtime_execution_observation_reads_plate_export_from_exact_owner( observation = RuntimeArtifactExecutionObservation.from_contexts( contexts, runtime_observations=runtime_observations ) - expectation = RuntimeArtifactExecutionExpectation.from_compiled_contexts(contexts) + expectation = RuntimeArtifactExecutionExpectation.from_executed_contexts(contexts) assert observation.exports.table_outputs == (contracted_output,) assert expectation.artifact_kinds == frozenset((MeasurementsArtifactType,)) diff --git a/tests/unit/test_runtime_value_store.py b/tests/unit/test_runtime_value_store.py index 4b95773301..429666d689 100644 --- a/tests/unit/test_runtime_value_store.py +++ b/tests/unit/test_runtime_value_store.py @@ -251,8 +251,8 @@ def test_output_plan_uses_its_single_scope_for_ungrouped_invocation(): resolved = output_plan.for_invocation_group(None) - assert resolved.group_keys == ("1",) - assert resolved.group_component is Microscopy.Channel + assert resolved.group_scope.keys == ("1",) + assert resolved.group_scope.component is Microscopy.Channel assert resolved.path == "/memory/Nuclei_1.pkl" @@ -267,8 +267,8 @@ def test_ungrouped_output_plan_ignores_incidental_invocation_group(): resolved = output_plan.for_invocation_group("1") - assert resolved.group_keys == (None,) - assert resolved.group_component is None + assert resolved.group_scope.keys == (None,) + assert resolved.group_scope.component is None assert resolved.path == "/memory/RGBImage.pkl" @@ -1809,7 +1809,7 @@ def _grouped_label_input_with_distinct_consumer_channel(): name=storage_plan.name, path=path, artifact_type=storage_plan.artifact_type, - group_scope=ComponentGroupScope.of(storage_plan.group_keys, component=storage_plan.group_component), + group_scope=ComponentGroupScope.of(storage_plan.group_scope.keys, component=storage_plan.group_scope.component), paths_by_group=storage_plan.paths_by_group, ).for_group("0") value = RuntimeValue.normalize_for_execution_scope( From 3622d71c98c021956bd62f97364a796765041b15 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 15:52:24 -0400 Subject: [PATCH 06/13] WIP: K7 tests migrated to context phases Co-Authored-By: Claude Opus 5.5 --- openhcs/core/orchestrator/worker_execution.py | 6 ++- tests/context_support.py | 41 ++++++++++++++++ .../test_compiled_artifact_inspection.py | 5 +- .../test_custom_source_transport.py | 5 +- tests/unit/agent/test_agent_services.py | 5 +- tests/unit/test_aligned_output_projection.py | 5 +- tests/unit/test_artifact_output_ownership.py | 5 +- tests/unit/test_artifact_plan_view.py | 5 +- .../unit/test_cellprofiler_analyst_export.py | 22 +++++---- ...test_cellprofiler_artifact_declarations.py | 5 +- ...cellprofiler_export_measurement_outputs.py | 14 ++++-- .../test_cellprofiler_export_to_database.py | 8 +++- .../test_cellprofiler_module_execution.py | 8 +++- ...ellprofiler_morphology_public_contracts.py | 5 +- .../test_cellprofiler_neighbors_contract.py | 11 +++-- ...profiler_runtime_callable_introspection.py | 5 +- tests/unit/test_cellprofiler_save_images.py | 7 ++- .../test_cellprofiler_spreadsheet_export.py | 5 +- tests/unit/test_compilation_session.py | 12 +++-- tests/unit/test_compiled_execution.py | 47 ++++++++++-------- ..._completed_output_publication_lifecycle.py | 48 ++++++++++++------- ...test_function_runtime_source_projection.py | 2 +- .../test_function_step_execution_scope.py | 20 ++++---- tests/unit/test_function_step_transport.py | 12 ++--- .../unit/test_invocation_contract_provider.py | 15 +++--- .../unit/test_k7_execution_protocol_guards.py | 11 ++--- .../test_loaded_pattern_group_ownership.py | 5 +- .../test_materialization_batch_outcomes.py | 47 ++++++++---------- .../unit/test_named_checkpoint_publication.py | 7 ++- .../test_opaque_main_flow_image_domain.py | 23 ++++++--- tests/unit/test_orchestrator_lane_planning.py | 11 +++-- .../unit/test_output_publication_ownership.py | 8 +++- .../unit/test_path_planner_materialization.py | 5 +- tests/unit/test_payload_axis_compile_gate.py | 14 ++++-- .../test_persisted_roi_only_publication.py | 22 +++++---- .../unit/test_runtime_context_declaration.py | 7 ++- .../unit/test_runtime_execution_validation.py | 28 ++++++----- ...test_runtime_source_resolution_snapshot.py | 5 +- .../unit/test_runtime_stack_cache_lifetime.py | 7 ++- tests/unit/test_source_binding_workspace.py | 16 ++++--- tests/unit/test_stream_component_semantics.py | 9 ++-- tests/unit/test_worker_start_policy.py | 7 ++- tests/unit/test_zmq_compilation.py | 5 +- tests/unit/test_zmq_execution_server.py | 14 ++++-- 44 files changed, 379 insertions(+), 195 deletions(-) create mode 100644 tests/context_support.py diff --git a/openhcs/core/orchestrator/worker_execution.py b/openhcs/core/orchestrator/worker_execution.py index c1b7cce080..9c9d1dfeaa 100644 --- a/openhcs/core/orchestrator/worker_execution.py +++ b/openhcs/core/orchestrator/worker_execution.py @@ -1351,10 +1351,12 @@ def _execute_axis_with_sequential_combinations( ) finally: # Required records and table projections now belong to the - # observation; the execution context and its stores end here. + # observation. This execution ends here: release its image cache + # and records now instead of waiting for the context to be collected. + context.release_execution_image_cache() if release_axis_resources: _release_runtime_resources((context,), owner=f"axis {axis_id}") - del context + context.runtime_value_store.clear() if observation.records or not observation.outputs.is_empty: runtime_observations.append(observation) diff --git a/tests/context_support.py b/tests/context_support.py new file mode 100644 index 0000000000..139936559c --- /dev/null +++ b/tests/context_support.py @@ -0,0 +1,41 @@ +"""Build processing contexts for tests without running the compiler.""" + +from __future__ import annotations + +import inspect + +from openhcs.core.context.processing_context import ( + CompiledContext, + CompilingContext, + ExecutingContext, +) + +_INIT_PARAMETERS = frozenset( + inspect.signature(CompilingContext.__init__).parameters +) - {"self"} + + +def compiled_context(**facts: object) -> CompiledContext: + """Return a compiled context holding exactly the given compiled facts.""" + + compiling = CompilingContext( + **{name: value for name, value in facts.items() if name in _INIT_PARAMETERS} + ) + unknown = facts.keys() - _INIT_PARAMETERS - vars(compiling).keys() + if unknown: + raise TypeError(f"Not compiled facts: {sorted(unknown)!r}.") + for name, value in facts.items(): + setattr(compiling, name, value) + return CompiledContext(vars(compiling)) + + +def executing_context(**facts: object) -> ExecutingContext: + """Return one execution of a context compiled from the given facts.""" + + return compiled_context(**facts).executing() + + +def with_facts(context: CompiledContext, **facts: object) -> ExecutingContext: + """Return a fresh execution of ``context`` with some compiled facts replaced.""" + + return CompiledContext(context.compiled_facts() | facts).executing() diff --git a/tests/integration/test_compiled_artifact_inspection.py b/tests/integration/test_compiled_artifact_inspection.py index 33f6ac307e..33dab83f17 100644 --- a/tests/integration/test_compiled_artifact_inspection.py +++ b/tests/integration/test_compiled_artifact_inspection.py @@ -68,6 +68,9 @@ StepCompletedProgressPhase, SuccessProgressStatus, ) +from tests.context_support import ( + executing_context, +) @contextmanager @@ -183,7 +186,7 @@ def produce(image): artifact_outputs=OrderedDict(((output_plan.ref(), output_plan),)), compiled_function_pattern=pattern, ) - context = ProcessingContext(step_plans={0: step_plan}, axis_id="A01") + context = executing_context(step_plans={0: step_plan}, axis_id="A01") bundle = CompiledExecutionBundle( pipeline_definition=(), runtime_contexts={"A01": context}, diff --git a/tests/integration/test_custom_source_transport.py b/tests/integration/test_custom_source_transport.py index 9863a0e07c..bfee92dff3 100644 --- a/tests/integration/test_custom_source_transport.py +++ b/tests/integration/test_custom_source_transport.py @@ -22,6 +22,9 @@ StepCompletedProgressPhase, SuccessProgressStatus, ) +from tests.context_support import ( + executing_context, +) WORKTREE = Path(__file__).resolve().parents[2] PYTHON = Path(sys.executable) @@ -477,7 +480,7 @@ def _compiled_custom_contexts(payload, runtime): step_index=0, step_name="Synthetic custom", axis_id=axis_id, compiled_function_pattern=pattern, ) - context = ProcessingContext( + context = executing_context( axis_id=axis_id, step_plans={0: plan}, filemanager=FileManager({Backend.MEMORY.value: MemoryStorageBackend()}), ) diff --git a/tests/unit/agent/test_agent_services.py b/tests/unit/agent/test_agent_services.py index da07ba1311..c54b554f79 100644 --- a/tests/unit/agent/test_agent_services.py +++ b/tests/unit/agent/test_agent_services.py @@ -125,6 +125,9 @@ CompileProgressPhase, RunningProgressStatus, ) +from tests.context_support import ( + executing_context, +) def sample_processing_function(image, sigma: float = 1.0): @@ -360,7 +363,7 @@ def _compile(self, request): paths_by_group={"A01": "/tmp/out/A01/objects.zarr"}, ) step_plan.artifact_outputs[output_plan.ref()] = output_plan - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") context.step_plans[0] = step_plan virtual_name = "A01_s001_w1_z001_t001.tif" full_virtual_path = str(request.plate / virtual_name) diff --git a/tests/unit/test_aligned_output_projection.py b/tests/unit/test_aligned_output_projection.py index 292fff2baa..2e9fddb35d 100644 --- a/tests/unit/test_aligned_output_projection.py +++ b/tests/unit/test_aligned_output_projection.py @@ -16,6 +16,9 @@ from openhcs.core.runtime_image_values import ImagePayload, ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.steps.function_runtime import PatternGroupExecutionRequest +from tests.context_support import ( + executing_context, +) def test_runtime_projects_mixed_output_planes_once_with_their_contexts(monkeypatch): @@ -45,7 +48,7 @@ def counted(payload): step_index=0, step_name="output-projection", axis_id="A01", output_memory_type=MemoryType.NUMPY, ), - context=ProcessingContext(axis_id="A01"), + context=executing_context(axis_id="A01"), compiled_group=compile_function_pattern( admitted_graph(lambda image: image), {}, diff --git a/tests/unit/test_artifact_output_ownership.py b/tests/unit/test_artifact_output_ownership.py index 471c2a7f4f..1169328f9a 100644 --- a/tests/unit/test_artifact_output_ownership.py +++ b/tests/unit/test_artifact_output_ownership.py @@ -36,6 +36,9 @@ CellProfilerRecordedArtifactOutputPolicy, ) from openhcs.processing.materialization import CsvOptions, MaterializationSpec +from tests.context_support import ( + executing_context, +) @pytest.mark.parametrize("invalid_policy", [None, False, ArtifactOutputPolicy]) @@ -385,7 +388,7 @@ def test_real_cellprofiler_declaration_compiles_without_a_table_wide_subject(): ), ) session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( axis_id="A01", step_plans={ 0: CompiledStepPlan( diff --git a/tests/unit/test_artifact_plan_view.py b/tests/unit/test_artifact_plan_view.py index 8747139209..90baa5be9c 100644 --- a/tests/unit/test_artifact_plan_view.py +++ b/tests/unit/test_artifact_plan_view.py @@ -65,6 +65,9 @@ StepCompletedProgressPhase, SuccessProgressStatus, ) +from tests.context_support import ( + executing_context, +) class QtApplication: @@ -140,7 +143,7 @@ def transform(image, labels): artifact_outputs=OrderedDict(((output_plan.ref(), output_plan),)), compiled_function_pattern=pattern, ) - context = ProcessingContext(step_plans={1: step_plan}, axis_id="A01") + context = executing_context(step_plans={1: step_plan}, axis_id="A01") bundle = CompiledExecutionBundle( pipeline_definition=(), runtime_contexts={"A01": context}, diff --git a/tests/unit/test_cellprofiler_analyst_export.py b/tests/unit/test_cellprofiler_analyst_export.py index 84b2242c50..1908c76ebb 100644 --- a/tests/unit/test_cellprofiler_analyst_export.py +++ b/tests/unit/test_cellprofiler_analyst_export.py @@ -100,6 +100,10 @@ from openhcs.processing.materialization import FileBundleOptions from openhcs.core.runtime_artifact_values import RuntimeValue from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, + with_facts, +) AXIS_ID = "A01_s1" RUNTIME_IMAGE_FIELD = MeasurementRowAxisField.SLICE_INDEX.value @@ -338,7 +342,7 @@ def source_workspace_metadata_document(self, _plate_path): def _export_context() -> ProcessingContext: - context = ProcessingContext( + context = executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -354,11 +358,11 @@ def _export_context() -> ProcessingContext: }, filemanager=SimpleNamespace(exists=lambda *_args: False), ) - context.dataset_root = Path("/") - context.microscope_handler = SimpleNamespace( + context = with_facts(context, dataset_root=Path("/")) + context = with_facts(context, microscope_handler=SimpleNamespace( metadata_handler=_MetadataHandlerStub(), source_bindings_still_required=lambda: None, - ) + )) return context @@ -888,18 +892,18 @@ def _borrowed_source_export_fixture( } } context = _export_context() - context.dataset_root = tmp_path - context.microscope_handler = SimpleNamespace( + context = with_facts(context, dataset_root=tmp_path) + context = with_facts(context, microscope_handler=SimpleNamespace( source_bindings_still_required=lambda: None, metadata_handler=SimpleNamespace( source_workspace_root=lambda _plate_path: tmp_path, source_workspace_metadata_document=lambda _plate_path: document, ) - ) - context.filemanager = SimpleNamespace( + )) + context = with_facts(context, filemanager=SimpleNamespace( exists=lambda *_args: False, source_path=lambda address, backend, *, base_path: address, - ) + )) plan = CompiledSourceBindingPlan( bindings=( NamedSourceBinding(alias="DNA"), diff --git a/tests/unit/test_cellprofiler_artifact_declarations.py b/tests/unit/test_cellprofiler_artifact_declarations.py index 645f877dc1..7b3bcf5a1d 100644 --- a/tests/unit/test_cellprofiler_artifact_declarations.py +++ b/tests/unit/test_cellprofiler_artifact_declarations.py @@ -88,6 +88,9 @@ from openhcs.processing.backends.cellprofiler.outlines import OverlayObjectsModule from openhcs.processing.backends.cellprofiler.morphology import MaskObjectsModule from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, +) def _module( @@ -175,7 +178,7 @@ def _compiler_contracts( resolved_steps.append(resolved_step) step_states[index] = step_state - context = ProcessingContext( + context = executing_context( step_plans={ index: CompiledStepPlan( step_index=index, diff --git a/tests/unit/test_cellprofiler_export_measurement_outputs.py b/tests/unit/test_cellprofiler_export_measurement_outputs.py index 09421e9d52..63246c600c 100644 --- a/tests/unit/test_cellprofiler_export_measurement_outputs.py +++ b/tests/unit/test_cellprofiler_export_measurement_outputs.py @@ -57,6 +57,10 @@ save_images_with_measurements, ) from openhcs.core.artifacts.plans import MAIN_FLOW +from tests.context_support import ( + executing_context, + with_facts, +) def _module(name: str, **settings: str) -> ModuleBlock: @@ -170,7 +174,7 @@ def _runtime_image_batch(*image_names: str) -> RuntimeArtifactBatch: def _export_context() -> ProcessingContext: - context = ProcessingContext( + context = executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -186,14 +190,14 @@ def _export_context() -> ProcessingContext: }, filemanager=SimpleNamespace(exists=lambda *_args: False), ) - context.dataset_root = Path("/") - context.microscope_handler = SimpleNamespace( + context = with_facts(context, dataset_root=Path("/")) + context = with_facts(context, microscope_handler=SimpleNamespace( source_bindings_still_required=lambda: None, metadata_handler=SimpleNamespace( source_workspace_root=Path, source_workspace_metadata_document=lambda _plate_path: None, ), - ) + )) return context @@ -343,7 +347,7 @@ def test_save_images_file_measurement_output_and_rows_are_conditional() -> None: 3 } - multi_axis_context = ProcessingContext(axis_id="A01") + multi_axis_context = executing_context(axis_id="A01") multi_axis_context.execution_runtime = SimpleNamespace( execution_axis_values=("A01", "A02") ) diff --git a/tests/unit/test_cellprofiler_export_to_database.py b/tests/unit/test_cellprofiler_export_to_database.py index f85efee94f..74ea371469 100644 --- a/tests/unit/test_cellprofiler_export_to_database.py +++ b/tests/unit/test_cellprofiler_export_to_database.py @@ -95,6 +95,10 @@ ExportToDatabaseModule, ) from openhcs.processing.backends.cellprofiler.relationships import RelateObjectsModule +from tests.context_support import ( + executing_context, + with_facts, +) OFFICIAL30_NATIVE_REFS = Path(__file__).parents[2] / "benchmark" / "native_refs" OFFICIAL30_DATABASE_PIPELINES = ( @@ -155,7 +159,7 @@ def _projection_builder() -> CellProfilerAnalystProjectionBuilder: def _export_context() -> ProcessingContext: - context = ProcessingContext( + context = executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -165,7 +169,7 @@ def _export_context() -> ProcessingContext: ) } ) - context.dataset_root = Path("/") + context = with_facts(context, dataset_root=Path("/")) return context diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index db73fc391b..5292195498 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -404,6 +404,10 @@ SliceAlignedLabels, ) from openhcs.core.artifacts.plans import MAIN_FLOW +from tests.context_support import ( + executing_context, + with_facts, +) def _compiled_callable_contract( @@ -9678,8 +9682,8 @@ def test_image_output_context_preserves_aligned_image_stack_payload(): def _pattern_group_runtime_for_output_memory( output_memory_type: str, ) -> PatternGroupExecutionRequest: - context = ProcessingContext(axis_id="A01") - context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + context = executing_context(axis_id="A01") + context = with_facts(context, microscope_handler=SimpleNamespace(parser=SourceSchemaFilenameParser())) return PatternGroupExecutionRequest( context=context, compiled_group=compile_function_pattern( diff --git a/tests/unit/test_cellprofiler_morphology_public_contracts.py b/tests/unit/test_cellprofiler_morphology_public_contracts.py index 1d65dcfd84..faf26b0a12 100644 --- a/tests/unit/test_cellprofiler_morphology_public_contracts.py +++ b/tests/unit/test_cellprofiler_morphology_public_contracts.py @@ -56,6 +56,9 @@ from openhcs.core.processing_contracts import ( Pure3DContract, ) +from tests.context_support import ( + executing_context, +) @dataclass(frozen=True, slots=True) @@ -192,7 +195,7 @@ def fixture_producer(image): steps = (producer_step, step) snapshots = tuple(current_step for index, current_step in enumerate(steps)) session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ index: CompiledStepPlan( step_index=index, diff --git a/tests/unit/test_cellprofiler_neighbors_contract.py b/tests/unit/test_cellprofiler_neighbors_contract.py index b082e9e6a6..82322cf749 100644 --- a/tests/unit/test_cellprofiler_neighbors_contract.py +++ b/tests/unit/test_cellprofiler_neighbors_contract.py @@ -58,6 +58,9 @@ from openhcs.core.processing_contracts import ( Pure3DContract, ) +from tests.context_support import ( + executing_context, +) def _neighbor_step() -> FunctionStep: @@ -106,7 +109,7 @@ def test_relationship_module_number_is_derived_after_public_transport() -> None: ) snapshot = restored_step session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -142,7 +145,7 @@ def test_relationship_module_number_is_derived_after_public_transport() -> None: def _compiled_neighbor_invocation(): step = _reconstructed_neighbor_step() snapshot = step - context = ProcessingContext( + context = executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -367,7 +370,7 @@ def fixture_producer(image): ) snapshots = tuple(step for index, step in enumerate(steps)) session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ index: CompiledStepPlan( step_index=index, @@ -511,7 +514,7 @@ def fixture_producer(image): ) snapshots = tuple(step for index, step in enumerate(steps)) session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ index: CompiledStepPlan( step_index=index, diff --git a/tests/unit/test_cellprofiler_runtime_callable_introspection.py b/tests/unit/test_cellprofiler_runtime_callable_introspection.py index 4a6e518bd2..6f6a9161e7 100644 --- a/tests/unit/test_cellprofiler_runtime_callable_introspection.py +++ b/tests/unit/test_cellprofiler_runtime_callable_introspection.py @@ -46,13 +46,16 @@ save_images, ) from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, +) def _compilation_session_for_steps( steps: list[FunctionStep], ) -> CompilationSession: return CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ index: CompiledStepPlan( step_index=index, diff --git a/tests/unit/test_cellprofiler_save_images.py b/tests/unit/test_cellprofiler_save_images.py index 854b458575..854791dd65 100644 --- a/tests/unit/test_cellprofiler_save_images.py +++ b/tests/unit/test_cellprofiler_save_images.py @@ -80,6 +80,9 @@ from openhcs.core.runtime_image_values import ImagePayload from tests.materialization_plans import output_plan as materialization_output_plan from openhcs.core.artifacts.plans import MAIN_FLOW +from tests.context_support import ( + executing_context, +) def _module(**settings: str) -> ModuleBlock: @@ -685,7 +688,7 @@ def test_save_images_isolates_declared_relative_paths_for_multi_axis_execution() ) assert materialization.write_mode is WriteMode.ERROR - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") context.execution_runtime = SimpleNamespace(execution_axis_values=("A01", "A02")) filemanager = FileManager({"memory": MemoryStorageBackend()}) primary_path = materialize( @@ -718,7 +721,7 @@ def test_save_images_isolates_declared_relative_paths_for_multi_axis_execution() def test_save_images_keeps_declared_relative_paths_for_single_axis_execution() -> None: scope = ExecutionAxisMaterializationRelativePathScope() - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") context.execution_runtime = SimpleNamespace(execution_axis_values=("A01",)) assert scope.scoped_path(PurePosixPath("exports/Saved.tiff"), context) == ( diff --git a/tests/unit/test_cellprofiler_spreadsheet_export.py b/tests/unit/test_cellprofiler_spreadsheet_export.py index 8a67108c1a..46aa7c7ef3 100644 --- a/tests/unit/test_cellprofiler_spreadsheet_export.py +++ b/tests/unit/test_cellprofiler_spreadsheet_export.py @@ -106,6 +106,9 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, ) +from tests.context_support import ( + executing_context, +) def test_export_to_spreadsheet_declares_exact_plate_callable_abi() -> None: contract = CallableContract.from_callable(export_to_spreadsheet) @@ -573,7 +576,7 @@ def test_export_to_spreadsheet_bundle_uses_generic_file_materialization() -> Non source_image_set_identity_policy=SourceImageSetIdentityPolicy(), ) filemanager = FileManager({"memory": MemoryStorageBackend()}) - context = ProcessingContext(filemanager=filemanager) + context = executing_context(filemanager=filemanager) with context.runtime_step_scope(): bundle = export_to_spreadsheet( add_filename_prefix=False, diff --git a/tests/unit/test_compilation_session.py b/tests/unit/test_compilation_session.py index bbc8220c45..7b9072434a 100644 --- a/tests/unit/test_compilation_session.py +++ b/tests/unit/test_compilation_session.py @@ -28,7 +28,10 @@ EMPTY_SOURCE_BINDINGS, StepSourceBindingsConfig, ) -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ( + CompilingContext, + ProcessingContext, +) from openhcs.core.function_patterns import ( compile_function_pattern, normalize_function_pattern, @@ -60,6 +63,9 @@ ) from openhcs.core.axes import Ungrouped from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, +) def _identity(image): @@ -183,7 +189,7 @@ def test_nonsequential_axis_compilation_finishes_its_initial_session(monkeypatch context = SimpleNamespace( pipeline_sequential_mode=False, pipeline_sequential_combinations=None, - freeze=lambda: events.append("freeze"), + freeze=lambda: events.append("freeze") or context, ) session = SimpleNamespace(context=context, global_config=GlobalPipelineConfig()) request = SimpleNamespace( @@ -318,7 +324,7 @@ def _compile_source_plans_for_contract( class _EffectiveConfigContextOrchestrator: def create_context(self, axis_id: str, *, resolved_config) -> ProcessingContext: del resolved_config - return ProcessingContext(axis_id=axis_id) + return CompilingContext(axis_id=axis_id) def test_axis_compilation_request_binds_post_execute_hooks_from_effective_config(): diff --git a/tests/unit/test_compiled_execution.py b/tests/unit/test_compiled_execution.py index 4ba86c3075..784afebcdc 100644 --- a/tests/unit/test_compiled_execution.py +++ b/tests/unit/test_compiled_execution.py @@ -56,6 +56,10 @@ from openhcs.core.component_group_scope import ( ComponentGroupScope, ) +from tests.context_support import ( + compiled_context, + executing_context, +) def _runtime_environment() -> CompiledRuntimeEnvironmentPlan: @@ -73,8 +77,8 @@ def _runtime_environment() -> CompiledRuntimeEnvironmentPlan: def test_compiled_execution_bundle_owns_transport_context_resolution(monkeypatch): - runtime_context = ProcessingContext(axis_id="A01") - resolved_context = ProcessingContext(axis_id="A01") + runtime_context = compiled_context(axis_id="A01") + resolved_context = compiled_context(axis_id="A01") resolver_calls = [] def resolve_for_transport(contexts): @@ -97,7 +101,7 @@ def resolve_for_transport(contexts): assert bundle.runtime_contexts == {"A01": runtime_context} transport_context = bundle.transport_contexts["A01"] assert transport_context is not resolved_context - assert transport_context.runtime_value_store is resolved_context.runtime_value_store + assert transport_context.axis_id == resolved_context.axis_id assert transport_context.step_plans is not resolved_context.step_plans assert bundle.worker_assignments == {"worker_0": ["A01"]} @@ -106,9 +110,9 @@ def test_compiled_execution_bundle_derives_axis_ids_from_contexts() -> None: bundle = CompiledExecutionBundle( pipeline_definition=(), runtime_contexts={ - "opaque-context-2": ProcessingContext(axis_id="B01"), - "opaque-context-1": ProcessingContext(axis_id="A01"), - "opaque-context-3": ProcessingContext(axis_id="A01"), + "opaque-context-2": compiled_context(axis_id="B01"), + "opaque-context-1": compiled_context(axis_id="A01"), + "opaque-context-3": compiled_context(axis_id="A01"), }, transport_contexts={}, worker_assignments={}, @@ -135,7 +139,7 @@ def test_compiled_execution_bundle_derives_runtime_observation_mode( execution_scopes, expected_mode, ) -> None: - context = ProcessingContext( + context = executing_context( axis_id="A01", step_plans={ index: SimpleNamespace( @@ -176,7 +180,7 @@ def test_consolidation_does_not_require_parent_payload_records( artifact_type=MeasurementsArtifactType, materialization=spec_type(CsvOptions()) if spec_type is not None else None, ) - context = ProcessingContext( + context = executing_context( axis_id="A01", post_execute_hooks=( AnalysisConsolidationHook( @@ -229,7 +233,7 @@ def test_axis_only_worker_lane_releases_runtime_values_after_each_axis( path="/memory/image.npy", artifact_type=ImageArtifactType, ) - contexts = [ProcessingContext(axis_id=axis) for axis in ("A01", "A02", "A03")] + contexts = [executing_context(axis_id=axis) for axis in ("A01", "A02", "A03")] payload_references = [] def execute_axis(_pipeline, context, _lane, **_kwargs): @@ -301,7 +305,7 @@ def test_runtime_execution_observation_merges_into_parent_contexts(): path="/memory/measurements.pkl", backend="memory", ) - parent_context = ProcessingContext(axis_id="A01") + parent_context = executing_context(axis_id="A01") RuntimeExecutionObservation( contexts=( @@ -374,7 +378,7 @@ def execute_current_run( execute_current_run, ) monkeypatch.setattr(progress_types, "_progress_queue", SimpleNamespace(put=lambda _event: None)) - context = ProcessingContext( + context = executing_context( axis_id="A01", step_plans={ 0: SimpleNamespace( @@ -412,14 +416,20 @@ def test_axis_teardown_preserves_execution_error_and_attempts_every_stage( ) -> None: cleaned = [] - def fail_step(*_args, **_kwargs): + pixels = np.zeros((16, 16), dtype=np.uint16) + pixel_reference = weakref.ref(pixels) + + def fail_step(_pipeline, context, *_args, **_kwargs): + context.runtime_image_stack_cache.store( + ("image",), memory_type="numpy", stack=pixels + ) raise RuntimeError("step failure") def fail_reset() -> None: raise RuntimeError("reset failure") assignment = FrameworkDeviceAssignment.from_mapping({MemoryType.CUPY: 2}) - context = ProcessingContext( + context = compiled_context( axis_id="A01", step_plans={ 0: SimpleNamespace( @@ -428,12 +438,6 @@ def fail_reset() -> None: ) }, ) - pixels = np.zeros((16, 16), dtype=np.uint16) - pixel_reference = weakref.ref(pixels) - context.runtime_image_stack_cache.store( - ("image",), memory_type="numpy", stack=pixels - ) - del pixels monkeypatch.setattr(progress_types, "_progress_queue", SimpleNamespace(put=lambda _event: None)) monkeypatch.setattr( worker_execution, @@ -465,6 +469,7 @@ def fail_reset() -> None: ) assert cleaned == [(MemoryType.CUPY, 2)] + del pixels assert pixel_reference() is None @@ -476,7 +481,7 @@ def test_worker_lane_releases_previous_axis_stack_before_next_axis( release_process_resources, ): references = [] - contexts = [ProcessingContext(axis_id=axis) for axis in ("A01", "A02", "A03")] + contexts = [executing_context(axis_id=axis) for axis in ("A01", "A02", "A03")] def execute_axis(_pipeline, context, _lane, **_kwargs): assert all(reference() is None for reference in references) @@ -522,7 +527,7 @@ def test_worker_lane_releases_unconsumed_image_records_before_next_axis( release_process_resources, ): references = [] - contexts = [ProcessingContext(axis_id=axis) for axis in ("A01", "A02", "A03")] + contexts = [executing_context(axis_id=axis) for axis in ("A01", "A02", "A03")] output_plan = ArtifactOutputPlan( name="processed", path="/memory/processed.pkl", diff --git a/tests/unit/test_completed_output_publication_lifecycle.py b/tests/unit/test_completed_output_publication_lifecycle.py index 4933475b43..0c54680b36 100644 --- a/tests/unit/test_completed_output_publication_lifecycle.py +++ b/tests/unit/test_completed_output_publication_lifecycle.py @@ -34,6 +34,10 @@ from openhcs.core.virtual_workspace_metadata import VirtualWorkspaceSourceProjectionEntries from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, + with_facts, +) def _facts(plate_root: Path) -> StepExecutionObservation: @@ -56,7 +60,7 @@ def _facts(plate_root: Path) -> StepExecutionObservation: def _context(plate_root: Path, count: int) -> ProcessingContext: - context = ProcessingContext( + context = executing_context( axis_id="A01", filemanager=SimpleNamespace(exists=lambda *_args: False), step_plans={index: SimpleNamespace( compiled_function_pattern=None, execution_scope=FunctionStepExecutionScope.AXIS, @@ -65,16 +69,15 @@ def _context(plate_root: Path, count: int) -> ProcessingContext: requires_main_flow_checkpoint=lambda _plans: False, ) for index in range(count)}, ) - context.dataset_root = plate_root - context.microscope_handler = SimpleNamespace( + context = with_facts(context, dataset_root=plate_root) + context = with_facts(context, microscope_handler=SimpleNamespace( parser=SourceSchemaFilenameParser(), metadata_handler=SimpleNamespace( source_workspace_metadata_document=lambda _p: None, source_workspace_root=Path, ), source_bindings_still_required=lambda: None, - ) - context.freeze() + )) return context @@ -114,7 +117,6 @@ def process(self, context, index): cancellation=signal, release_axis_resources=False, ) assert type(result) is (ExecutionCancelled if cancel else ExecutionFailed) - assert not context.runtime_value_store.observed_values assert pixel_refs[0]() is None assert len(result.runtime_observation.contexts) == 1 observation = result.runtime_observation.contexts[0] @@ -122,7 +124,7 @@ def process(self, context, index): assert observation.outputs.source_projection_entries_by_target == facts.source_projection_entries_by_target restored = pickle.loads(pickle.dumps(result)) assert restored.runtime_observation.contexts[0].outputs.source_projection_entries_by_target == facts.source_projection_entries_by_target - parent = ProcessingContext(axis_id="A01") + parent = executing_context(axis_id="A01") restored.runtime_observation.merge_into({"A01": parent}) assert parent.completed_step_outputs.source_projection_entries_by_target == facts.source_projection_entries_by_target @@ -143,8 +145,11 @@ def test_same_plate_reader_uses_completed_facts_before_durable_publication(tmp_p filemanager=context.filemanager, ) assert direct.projection_if_available() is None - context.bind_execution_runtime(_lane()) - assert authority.projection_if_available() is None + next_execution = context.executing(_lane()) + assert ( + next_execution.runtime_source_workspace_projections.projection_if_available() + is None + ) def test_runtime_overlay_excludes_another_output_plate_and_other_context(tmp_path): @@ -152,9 +157,11 @@ def test_runtime_overlay_excludes_another_output_plate_and_other_context(tmp_pat context.record_completed_step_outputs(_facts(tmp_path / "other-plate")) authority = context.runtime_source_workspace_projections assert authority.projection_if_available() is None - other = _context(tmp_path, 1) - other.microscope_handler = context.microscope_handler - other.filemanager = context.filemanager + other = with_facts( + _context(tmp_path, 1), + microscope_handler=context.microscope_handler, + filemanager=context.filemanager, + ) assert authority.is_bound_to_context(context) assert not authority.is_bound_to_context(other) @@ -177,9 +184,16 @@ def test_runtime_axis_query_is_explicit_and_full_plate_outputs_remain_available( assert set(full.pipeline_start_files()) == {str(tmp_path / "images/A01.tif"), str(tmp_path / "images/A02.tif")} assert axis.pipeline_start_files() == (str(tmp_path / "images/A01.tif"),) # Another context shares declaration admission, never the completed facts. - other = _context(tmp_path, 1) - other.filemanager = context.filemanager - other.microscope_handler = context.microscope_handler + other = with_facts( + _context(tmp_path, 1), + filemanager=context.filemanager, + microscope_handler=context.microscope_handler, + ) assert other.runtime_source_workspace_projections.projection_if_available() is None - context.reset_completed_step_outputs() - assert authority.projection_if_available(axis_id="A01") is None + # Nor does the next execution of the same compiled context. + assert ( + context.executing().runtime_source_workspace_projections.projection_if_available( + axis_id="A01" + ) + is None + ) diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index 9aaf55246d..be92abddab 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -2226,7 +2226,7 @@ def test_first_step_prepares_raw_source_anchors_under_semantic_binding_groups( finally: set_progress_queue(None) - context = compilation.runtime_contexts["A01"] + context = compilation.runtime_contexts["A01"].executing() executor = FunctionStepExecutor(context, 0) grouped_patterns = executor._prepare_groups(executor._detect_patterns()) diff --git a/tests/unit/test_function_step_execution_scope.py b/tests/unit/test_function_step_execution_scope.py index 45c54a004f..1cff072025 100644 --- a/tests/unit/test_function_step_execution_scope.py +++ b/tests/unit/test_function_step_execution_scope.py @@ -97,13 +97,17 @@ StepCompletedProgressPhase, StepStartedProgressPhase, ) +from tests.context_support import ( + executing_context, + with_facts, +) def test_runtime_step_values_share_within_step_and_release_between_steps() -> None: class Counter: pass - context = ProcessingContext() + context = executing_context() with pytest.raises(RuntimeError, match="active FunctionStep"): context.runtime_step_value(Counter) @@ -438,13 +442,13 @@ def _plate_context( axis_id: str, plans: tuple[CompiledStepPlan, ...], ) -> ProcessingContext: - context = ProcessingContext( + context = executing_context( step_plans={plan.step_index: plan for plan in plans}, axis_id=axis_id, filemanager=FileManager({Backend.MEMORY.value: MemoryStorageBackend()}), post_execute_hooks=PostExecuteHook.bind_all(GlobalPipelineConfig()), ) - context.microscope_handler = SimpleNamespace() + context = with_facts(context, microscope_handler=SimpleNamespace()) return context @@ -925,10 +929,10 @@ def export(*, artifact_batch: RuntimeArtifactBatch): plan.output_dir = tmp_path / "images" plan.analysis_results_dir = str(tmp_path / "analysis") context = _plate_context("A01", (plan,)) - context.metadata_cache = {} - context.microscope_handler = SimpleNamespace( + context = with_facts(context, metadata_cache={}) + context = with_facts(context, microscope_handler=SimpleNamespace( parser=SourceSchemaFilenameParser(), source_name="test", - ) + )) _record_measurements( context, name="prior_measurements", @@ -1000,9 +1004,9 @@ def export(*, artifact_batch: RuntimeArtifactBatch): metadata_writer=True, ) context = _plate_context("A01", (plan,)) - context.source_image_set_identity_policy = SourceImageSetIdentityPolicy( + context = with_facts(context, source_image_set_identity_policy=SourceImageSetIdentityPolicy( frozenset((Microscopy.Channel,)) - ) + )) _record_measurements( context, name=measurement_spec.name, diff --git a/tests/unit/test_function_step_transport.py b/tests/unit/test_function_step_transport.py index b10247a618..45b1db6891 100644 --- a/tests/unit/test_function_step_transport.py +++ b/tests/unit/test_function_step_transport.py @@ -43,6 +43,10 @@ ZMQExecutionRequestBuilder, ) from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + compiled_context, + executing_context, +) def _execute_spawned_custom_invocation( @@ -432,11 +436,10 @@ def test_bundle_transport_preserves_prepared_runtime_contract_identity() -> None step_index=0, step_name="Crop", axis_id="A01", compiled_function_pattern=pattern, ) - context = ProcessingContext( + context = compiled_context( axis_id="A01", step_plans={0: plan}, filemanager=FileManager({Backend.MEMORY.value: MemoryStorageBackend()}), ) - context.freeze() second_invocation = replace( invocation, kwargs=(("left_right_rectangle_positions", (2, 5)),), ) @@ -444,10 +447,9 @@ def test_bundle_transport_preserves_prepared_runtime_contract_identity() -> None pattern, groups=(replace(pattern.groups[0], invocations=(second_invocation,)),), ) second_plan = replace(plan, axis_id="A02", compiled_function_pattern=second_pattern) - second_context = ProcessingContext( + second_context = compiled_context( axis_id="A02", step_plans={0: second_plan}, filemanager=context.filemanager, ) - second_context.freeze() contexts = {"A01": context, "A02": second_context} bundle = CompiledExecutionBundle.from_runtime_contexts( pipeline_definition=(), runtime_contexts=contexts, @@ -470,8 +472,6 @@ def test_bundle_transport_preserves_prepared_runtime_contract_identity() -> None assert second_transport_invocation.kwargs == second_invocation.kwargs assert plan.compiled_function_pattern is pattern assert invocation.contract.resolve_runtime_callable() is prepared_callable - assert transport_context.runtime_value_store is context.runtime_value_store - assert transport_context.runtime_image_stack_cache is context.runtime_image_stack_cache assert transport_context.filemanager is context.filemanager transport_plan.step_name = "Transport-only name" assert plan.step_name == "Crop" diff --git a/tests/unit/test_invocation_contract_provider.py b/tests/unit/test_invocation_contract_provider.py index ed8207c757..63c63ac3e0 100644 --- a/tests/unit/test_invocation_contract_provider.py +++ b/tests/unit/test_invocation_contract_provider.py @@ -43,6 +43,9 @@ Pure3DContract, ) from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, +) def _identity(image, *, sigma: float = 1.0): @@ -703,7 +706,7 @@ def test_cellprofiler_provider_reconstructs_exact_contract_from_public_step() -> ) snapshot = step session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -788,7 +791,7 @@ def test_cellprofiler_provider_advances_native_artifacts_through_generic_graph() for index, step in enumerate(steps) } session = CompilationSession.from_context( - context=ProcessingContext(step_plans=plans, axis_id="A01"), + context=executing_context(step_plans=plans, axis_id="A01"), orchestrator=SimpleNamespace(pipeline_config=PipelineConfig()), global_config=GlobalPipelineConfig(), pipeline=ResolvedPipelineDefinition( @@ -835,7 +838,7 @@ def test_cellprofiler_provider_leaves_native_same_name_callable_unclaimed() -> N step = FunctionStep(func=crop, name="Native NumPy crop") snapshot = step session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -1141,7 +1144,7 @@ def test_cellprofiler_provider_rejects_under_specified_one_image_align() -> None ) snapshot = step session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -1280,7 +1283,7 @@ def test_calculate_math_provider_keeps_object_identity_and_output_name_public() ) steps = (measurement_step, step) session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ index: CompiledStepPlan( step_index=index, @@ -1372,7 +1375,7 @@ def session_for( ) return ( CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ index: CompiledStepPlan( step_index=index, diff --git a/tests/unit/test_k7_execution_protocol_guards.py b/tests/unit/test_k7_execution_protocol_guards.py index 335c1b82f6..35777198fd 100644 --- a/tests/unit/test_k7_execution_protocol_guards.py +++ b/tests/unit/test_k7_execution_protocol_guards.py @@ -63,7 +63,7 @@ def _concrete_subclasses(base: type) -> set[type]: - """Leaf members of a family, as their modules publish them.""" + """Instantiable members of a family, as their modules publish them.""" found: set[type] = set() pending = list(base.__subclasses__()) while pending: @@ -74,10 +74,6 @@ def _concrete_subclasses(base: type) -> set[type]: cls.__module__.startswith("openhcs.") and getattr(module, cls.__name__, None) is cls and not inspect.isabstract(cls) - and not any( - getattr(module, sub.__name__, None) is sub - for sub in cls.__subclasses__() - ) ): found.add(cls) return found @@ -131,7 +127,10 @@ def test_context_phases_and_results_are_the_declared_families() -> None: CompiledContext, ExecutingContext, } - assert _concrete_subclasses(ExecutionResult) == { + # ExecutionStopped is the shared base of the two stopped outcomes. + assert _concrete_subclasses(ExecutionResult) - { + execution_result.ExecutionStopped + } == { ExecutionSucceeded, ExecutionFailed, ExecutionCancelled, diff --git a/tests/unit/test_loaded_pattern_group_ownership.py b/tests/unit/test_loaded_pattern_group_ownership.py index 05c9b8974e..57e22fd428 100644 --- a/tests/unit/test_loaded_pattern_group_ownership.py +++ b/tests/unit/test_loaded_pattern_group_ownership.py @@ -31,6 +31,9 @@ from openhcs.core.steps.function_output_manifest import NoStepOutputManifestMatch from openhcs.domains.microscopy.axes import Microscopy from openhcs.core.runtime_image_values import PlainImagePayload +from tests.context_support import ( + executing_context, +) def _identity(image): @@ -50,7 +53,7 @@ def _fixture(): variable_components=(Microscopy.Site,), source_binding_plan=CompiledSourceBindingPlan(), ) - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") request = PatternGroupExecutionRequest( context=context, execution_plan=plan, diff --git a/tests/unit/test_materialization_batch_outcomes.py b/tests/unit/test_materialization_batch_outcomes.py index 969639b7e7..1b251698e2 100644 --- a/tests/unit/test_materialization_batch_outcomes.py +++ b/tests/unit/test_materialization_batch_outcomes.py @@ -43,6 +43,9 @@ ) from openhcs.processing.materialization.core import _WRITERS_BY_OPTIONS from tests.materialization_plans import output_plan as materialization_output_plan +from tests.context_support import ( + compiled_context, +) class CsvOnlyMemoryBackend(MemoryStorageBackend): @@ -137,30 +140,23 @@ def process(self, context, step_index): return StepExecutionObservation(MappingProxyType({}), (self.path,)) -class FrozenWorkerContext(ProcessingContext): - def __init__(self): - super().__init__( - axis_id="A01", - filemanager=FileManager({"memory": MemoryStorageBackend()}), - step_plans={ - 0: CompiledStepPlan( - step_index=0, - step_name="Save", - axis_id="A01", - compiled_function_pattern=compile_function_pattern( - admitted_graph(lambda image: image), - {}, - {}, - ), - ) - }, - ) - self.freeze() - self.released = False - - def release_execution_image_cache(self): - super().release_execution_image_cache() - self.released = True +def _compiled_worker_context(): + return compiled_context( + axis_id="A01", + filemanager=FileManager({"memory": MemoryStorageBackend()}), + step_plans={ + 0: CompiledStepPlan( + step_index=0, + step_name="Save", + axis_id="A01", + compiled_function_pattern=compile_function_pattern( + admitted_graph(lambda image: image), + {}, + {}, + ), + ) + }, + ) @pytest.mark.parametrize("mode", tuple(RuntimeObservationMode.__subclasses__())) @@ -170,7 +166,7 @@ def test_worker_exports_actual_step_outcomes_after_resource_release( import openhcs.core.orchestrator.worker_execution as worker path = tmp_path / "actual.csv" - context = FrozenWorkerContext() + context = _compiled_worker_context() lane = WorkerLaneExecutionContext( execution_id="execution", subject_id="plate", @@ -197,7 +193,6 @@ def forbidden_reconstruction(*_args, **_kwargs): ) transported = pickle.loads(pickle.dumps(result)) - assert context.released assert transported.is_success() assert transported.runtime_observation.contexts[0].outputs.runtime_export_paths == (path,) assert transported.runtime_observation.contexts[0].records == () diff --git a/tests/unit/test_named_checkpoint_publication.py b/tests/unit/test_named_checkpoint_publication.py index a6cf024341..bfbb747d75 100644 --- a/tests/unit/test_named_checkpoint_publication.py +++ b/tests/unit/test_named_checkpoint_publication.py @@ -100,12 +100,15 @@ def test_named_and_ordinary_checkpoint_inventory_owns_every_address(tmp_path, ma step_materialization_config=LazyStepMaterializationConfig(enabled=True), ) bundle = orchestrator.compile_pipelines([step]) + contexts = { + key: context.executing() for key, context in bundle.runtime_contexts.items() + } observations = RuntimeExecutionObservation(contexts=tuple( RuntimeContextObservation(context_key=key, records=(), outputs=step.process(context, 0)) - for key, context in bundle.runtime_contexts.items() + for key, context in contexts.items() )) OpenHCSMetadataTarget.finalize_completed_dataset( - bundle.runtime_contexts, runtime_observations=(observations,) + contexts, runtime_observations=(observations,) ) root = tmp_path / "source_out" metadata = json.loads((root / "openhcs_metadata.json").read_text()) diff --git a/tests/unit/test_opaque_main_flow_image_domain.py b/tests/unit/test_opaque_main_flow_image_domain.py index 3589506938..266865185a 100644 --- a/tests/unit/test_opaque_main_flow_image_domain.py +++ b/tests/unit/test_opaque_main_flow_image_domain.py @@ -29,6 +29,10 @@ from openhcs.core.payload_axes import PayloadAxes from openhcs.core.axes import ColourAxis from openhcs.core.runtime_image_values import ImagePayload +from tests.context_support import ( + executing_context, + with_facts, +) def _identity(image): @@ -38,7 +42,7 @@ def _identity(image): def _runtime(): pattern = compile_function_pattern(admitted_graph(_identity), {}, {}) runtime = PatternGroupExecutionRequest( - context=ProcessingContext(axis_id="A01"), + context=executing_context(axis_id="A01"), pattern_group_info="", component_index=0, component_count=1, compiled_group=pattern.default_group, execution_plan=CompiledStepPlan( @@ -132,8 +136,13 @@ def test_nominal_alignment_owner_preserves_single_output_topology(named, axis): @pytest.mark.parametrize("count", [1, 3]) def test_raw_array_fallback_keeps_explicit_runtime_unstack_contract(count): runtime = _runtime() - runtime = replace(runtime, context=ProcessingContext(axis_id="A01")) - runtime.context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + runtime = replace( + runtime, + context=executing_context( + axis_id="A01", + microscope_handler=SimpleNamespace(parser=SourceSchemaFilenameParser()), + ), + ) pixels = ImagePayload.of(np.arange(count * 4 * 5, dtype=np.float32).reshape(count, 4, 5)) projected = runtime._project_output_slices(pixels, [f"source-{i}.tif" for i in range(count)]) payloads = tuple(payload for payload, _context in projected) @@ -193,8 +202,8 @@ def test_save_and_next_load_preserve_domain_and_independent_cache(tmp_path, monk plan = replace(producer.execution_plan, output_dir=tmp_path, step_scope_id="whole-producer", compiled_function_pattern=pattern) producer = replace(producer, execution_plan=plan) files = _MemoryFiles() - context = ProcessingContext(axis_id="A01", filemanager=files) - context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + context = executing_context(axis_id="A01", filemanager=files) + context = with_facts(context, microscope_handler=SimpleNamespace(parser=SourceSchemaFilenameParser())) producer = replace(producer, context=context) producer = replace(producer, pattern_group_info="whole") output = ( @@ -350,8 +359,8 @@ def test_saved_mixed_named_cohort_rejects_joint_load_and_preserves_selected_doma plan = replace(producer.execution_plan, output_dir=tmp_path, step_scope_id="mixed-producer", compiled_function_pattern=pattern) producer = replace(producer, execution_plan=plan) files = _MemoryFiles() - context = ProcessingContext(axis_id="A01", filemanager=files) - context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + context = executing_context(axis_id="A01", filemanager=files) + context = with_facts(context, microscope_handler=SimpleNamespace(parser=SourceSchemaFilenameParser())) producer = replace(producer, context=context) producer = replace(producer, pattern_group_info="mixed") diff --git a/tests/unit/test_orchestrator_lane_planning.py b/tests/unit/test_orchestrator_lane_planning.py index df50215047..8c7cbf6961 100644 --- a/tests/unit/test_orchestrator_lane_planning.py +++ b/tests/unit/test_orchestrator_lane_planning.py @@ -80,6 +80,9 @@ SuccessProgressPhase, ViewerSettlementProgressPhase, ) +from tests.context_support import ( + executing_context, +) PROGRESS_CONTEXT = ProgressExecutionContext( execution_id="exec", @@ -209,7 +212,7 @@ def test_fork_resources_join_all_lanes_after_plate_failure(monkeypatch, failure) def _compiled_context(axis_id: str) -> ProcessingContext: - return ProcessingContext(axis_id=axis_id) + return executing_context(axis_id=axis_id) def _runtime_environment( @@ -1242,7 +1245,7 @@ def rollback_failed_bootstrap(self): ), ) - def acquire(config, _visualizer_config): + def acquire(config): if config is second_config: raise RuntimeError("second viewer failed") return AcquiredViewer() @@ -1250,7 +1253,6 @@ def acquire(config, _visualizer_config): orchestrator = SimpleNamespace(get_or_create_visualizer=acquire) context = SimpleNamespace( required_visualizers=required_visualizers, - visualizer_config=None, ) with pytest.raises(RuntimeError, match="second viewer failed"): @@ -1279,11 +1281,10 @@ def test_viewer_bootstrap_rolls_back_after_post_launch_validation_failure( launch_message="Launching viewer", ) orchestrator = SimpleNamespace( - get_or_create_visualizer=lambda _config, _visualizer_config: visualizer, + get_or_create_visualizer=lambda _config: visualizer, ) context = SimpleNamespace( required_visualizers=(required_visualizer,), - visualizer_config=None, ) monkeypatch.setattr( compiled_plate_execution_module, diff --git a/tests/unit/test_output_publication_ownership.py b/tests/unit/test_output_publication_ownership.py index 8d3c239f3a..1b9f4f60c2 100644 --- a/tests/unit/test_output_publication_ownership.py +++ b/tests/unit/test_output_publication_ownership.py @@ -27,6 +27,10 @@ from openhcs.core.steps.function_runtime import PatternGroupExecutionRequest from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser from openhcs.core.runtime_image_values import ImagePayload +from tests.context_support import ( + executing_context, + with_facts, +) SOURCE = "/source/A01_s001_w1_z001_t001.tif" @@ -38,8 +42,8 @@ def _identity(image): def _runtime(tmp_path): filemanager = FileManager({"memory": MemoryStorageBackend()}) - context = ProcessingContext(axis_id="A01", filemanager=filemanager) - context.microscope_handler = SimpleNamespace(parser=SourceSchemaFilenameParser()) + context = executing_context(axis_id="A01", filemanager=filemanager) + context = with_facts(context, microscope_handler=SimpleNamespace(parser=SourceSchemaFilenameParser())) plan = CompiledStepPlan( step_index=0, step_name="OutputOwnership", diff --git a/tests/unit/test_path_planner_materialization.py b/tests/unit/test_path_planner_materialization.py index d91dffd4e7..d79a9b28ff 100644 --- a/tests/unit/test_path_planner_materialization.py +++ b/tests/unit/test_path_planner_materialization.py @@ -125,6 +125,9 @@ from openhcs.processing.backends.analysis.metaxpress_utils import HiddenPixelSize from openhcs.core.axes import Axis, GroupingDeclaration, Ungrouped from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, +) def _execute_compiled_metadata_pattern(compiled, input_plans=None, stored_outputs=()): @@ -1193,7 +1196,7 @@ def test_implicit_native_main_flow_provenance_drives_artifact_owned_scope(): ), ) session = CompilationSession.from_context( - context=ProcessingContext( + context=executing_context( step_plans={ index: CompiledStepPlan( step_index=index, diff --git a/tests/unit/test_payload_axis_compile_gate.py b/tests/unit/test_payload_axis_compile_gate.py index a3802050af..2e66fd2583 100644 --- a/tests/unit/test_payload_axis_compile_gate.py +++ b/tests/unit/test_payload_axis_compile_gate.py @@ -17,7 +17,10 @@ from openhcs.core.artifacts.kinds import ImageArtifactType from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.component_group_scope import ComponentGroupScope -from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.context.processing_context import ( + CompilingContext, + ProcessingContext, +) from openhcs.core.function_patterns import ( MainFlowInputProjection, CompiledFunctionGroup, @@ -41,6 +44,9 @@ from openhcs.processing.backends.cellprofiler.color import color_to_gray from openhcs.processing.backends.cellprofiler.crop import crop from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, +) class _CarrierTestSession(SimpleNamespace): @@ -143,7 +149,7 @@ def _session( source_binding_plan=CompiledSourceBindingPlan(bindings=(binding,)), compiled_function_pattern=_compiled_pattern(), ) - context = ProcessingContext( + context = CompilingContext( step_plans={0: plan}, axis_id="A01", filemanager=FileManager(dict(storage_registry)), @@ -285,7 +291,7 @@ def identity(image): compiled_function_pattern=pattern, ) ensure_storage_registry() - context = ProcessingContext( + context = CompilingContext( step_plans={0: plan}, axis_id="A01", filemanager=FileManager(dict(storage_registry)), @@ -650,7 +656,7 @@ def test_unknown_routed_group_cannot_fall_back_to_all_primary_sources( ensure_storage_registry() session = _CarrierTestSession( plans={0: plan}, - context=ProcessingContext( + context=CompilingContext( step_plans={0: plan}, axis_id="A01", filemanager=FileManager(dict(storage_registry)), diff --git a/tests/unit/test_persisted_roi_only_publication.py b/tests/unit/test_persisted_roi_only_publication.py index a14cbb7ceb..81b46d3bec 100644 --- a/tests/unit/test_persisted_roi_only_publication.py +++ b/tests/unit/test_persisted_roi_only_publication.py @@ -46,6 +46,10 @@ from openhcs.processing.materialization.options import FileBundleOptions from openhcs.domains.microscopy.axes import Microscopy from tests.materialization_plans import output_plan as materialization_output_plan +from tests.context_support import ( + executing_context, + with_facts, +) def _publish_saved_step(context, plan, *, artifact_materializations=()): @@ -70,11 +74,11 @@ def _publish_saved_step(context, plan, *, artifact_materializations=()): @pytest.fixture def publication_context(tmp_path): filemanager = FileManager({Backend.DISK.value: DiskStorageBackend()}) - context = ProcessingContext(filemanager=filemanager) - context.metadata_cache = {} - context.microscope_handler = SimpleNamespace( + context = executing_context(filemanager=filemanager) + context = with_facts(context, metadata_cache={}) + context = with_facts(context, microscope_handler=SimpleNamespace( parser=SourceSchemaFilenameParser(), source_name="openhcsdata" - ) + )) context.publication_audit = [] return context @@ -236,7 +240,7 @@ def test_saved_roi_only_batch_publishes_and_reconciles_without_rendering( assert published["subdirectories"][destination]["source_projection"] == [] # Final reconciliation has no runtime outcome/value to reconstruct. Its # destination comes from the same durable declaration the reader admitted. - publication_context.step_plans = {0: plan} + publication_context = with_facts(publication_context, step_plans={0: plan}) owner = RuntimeArtifactMetadataTarget.from_plan(plan) reconciled = tuple( target.output_dir @@ -255,12 +259,12 @@ def test_result_only_destination_saved_through_zarr_passthrough_publishes_comple """Zarr saves archives through disk passthrough; publication must still see them.""" from polystore.zarr import ZarrStorageBackend - publication_context.filemanager = FileManager( + publication_context = with_facts(publication_context, filemanager=FileManager( { Backend.DISK.value: DiskStorageBackend(), Backend.ZARR.value: ZarrStorageBackend(), } - ) + )) monkeypatch.setattr( "polystore.backend_registry.get_backend_instance", lambda key: publication_context.filemanager._get_backend(key), @@ -270,7 +274,7 @@ def test_result_only_destination_saved_through_zarr_passthrough_publishes_comple publication_context, plate, "results", monkeypatch, backend=Backend.ZARR.value ) plan = plan_for(plate, Backend.ZARR.value) - publication_context.step_plans = {0: plan} + publication_context = with_facts(publication_context, step_plans={0: plan}) _publish_saved_step(publication_context, plan, artifact_materializations=(outcome,)) OpenHCSMetadataTarget.finalize_completed_dataset({"A01": publication_context}) @@ -356,7 +360,7 @@ def test_measurement_only_step_does_not_reconcile_pending_image_producer( metadata_path, destination, SourceProjectionMetadataSerializer.projection_fields(((projection, virtual_path),)), ) - publication_context.step_plans = {0: plan} + publication_context = with_facts(publication_context, step_plans={0: plan}) OpenHCSMetadataTarget.finalize_completed_dataset({"A01": publication_context}) reconciled = json.loads(metadata_path.read_text())[FIELDS.SUBDIRECTORIES][destination] assert reconciled[FIELDS.IMAGE_FILES] == [virtual_path] diff --git a/tests/unit/test_runtime_context_declaration.py b/tests/unit/test_runtime_context_declaration.py index 2adacd15b7..3409a63e94 100644 --- a/tests/unit/test_runtime_context_declaration.py +++ b/tests/unit/test_runtime_context_declaration.py @@ -34,6 +34,9 @@ from openhcs.core.processing_contracts import ( Pure2DContract, ) +from tests.context_support import ( + executing_context, +) def _inferred_context(image, *, context=None): @@ -127,7 +130,7 @@ def process(image, *, context=None): ) def test_runtime_binding_consumes_captured_selection(func, parameter): pattern = compile_function_pattern(admitted_graph(func), {}, {}) - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") artifacts = ({}, {}) scope = PatternGroupData( matching_files=["input.tif"], @@ -169,7 +172,7 @@ def test_save_images_variants_keep_independent_injection_declarations(): def test_measurement_variant_keeps_real_axis_scoped_filename_context(): - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") context.execution_runtime = SimpleNamespace(execution_axis_values=("A01", "A02")) payload = ImagePayloadMetadata(source_path="/input/DNA.tif").payload_with( np.ones((2, 3), dtype=np.uint16) diff --git a/tests/unit/test_runtime_execution_validation.py b/tests/unit/test_runtime_execution_validation.py index c60ed82f59..f98254e764 100644 --- a/tests/unit/test_runtime_execution_validation.py +++ b/tests/unit/test_runtime_execution_validation.py @@ -74,6 +74,10 @@ ZMQRuntimeExecutionObservationExport, ) from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, + with_facts, +) def _compiled_pattern( @@ -107,22 +111,22 @@ def _streaming_context( step_plans: dict[int, CompiledStepPlan], ) -> ProcessingContext: ensure_storage_registry() - context = ProcessingContext( + context = executing_context( step_plans=step_plans, axis_id="A01", filemanager=FileManager(dict(storage_registry)), ) - context.microscope_handler = SimpleNamespace( + context = with_facts(context, microscope_handler=SimpleNamespace( parser=SourceSchemaFilenameParser(), source_name="runtime_validation", - ) - context.dataset_root = Path("/tmp/plate") + )) + context = with_facts(context, dataset_root=Path("/tmp/plate")) return context def test_runtime_execution_validation_detects_missing_artifact_kind() -> None: observation = RuntimeArtifactExecutionObservation.from_contexts( - {"A01": ProcessingContext(axis_id="A01")}, + {"A01": executing_context(axis_id="A01")}, runtime_observations=(), ) @@ -146,7 +150,7 @@ def test_runtime_execution_validation_detects_missing_artifact_kind() -> None: def test_runtime_execution_observation_reads_context_stores() -> None: - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") context.runtime_value_store.record( RuntimeValue( key=ArtifactKey( @@ -182,7 +186,7 @@ def test_runtime_execution_observation_ignores_uncontracted_files( table_output.write_text("ObjectNumber,Area\n1,42\n", encoding="utf-8") image_output.write_bytes(b"not decoded by export observation") - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") observation = RuntimeArtifactExecutionObservation.from_contexts( {"A01": context}, runtime_observations=() @@ -208,7 +212,7 @@ def test_zmq_observation_exports_exact_compiler_owned_artifacts( artifact_type=MeasurementsArtifactType, materialization=materialization, ) - context = ProcessingContext( + context = executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -233,7 +237,7 @@ def test_zmq_observation_exports_exact_compiler_owned_artifacts( ) export = ZMQRuntimeExecutionObservationExport.from_execution( - compiled_contexts={"A01": context}, + executed_contexts={"A01": context}, execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path,), runtime_observations=runtime_observations, @@ -485,7 +489,7 @@ def test_runtime_execution_observation_reads_plate_export_from_exact_owner( ) pattern = _compiled_pattern(FunctionStepExecutionScope.PLATE) contexts = { - axis_id: ProcessingContext( + axis_id: executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -536,7 +540,7 @@ def test_runtime_execution_observation_reads_plate_export_from_exact_owner( def test_zmq_observation_compresses_and_preserves_exact_runtime_records( tmp_path, ) -> None: - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") context.runtime_value_store.record( RuntimeValue( key=ArtifactKey( @@ -560,7 +564,7 @@ def test_zmq_observation_compresses_and_preserves_exact_runtime_records( backend="memory", ) export = ZMQRuntimeExecutionObservationExport.from_execution( - compiled_contexts={"A01": context}, + executed_contexts={"A01": context}, execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(tmp_path,), runtime_observations=(), diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py index 87198f3581..709dc3a946 100644 --- a/tests/unit/test_runtime_source_resolution_snapshot.py +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -29,6 +29,9 @@ from openhcs.core.dataset_sources.interfaces import FilenameParseResult from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, +) PATH = "A01_s001_w1_z001_t001.tif" @@ -308,7 +311,7 @@ def test_resolved_direct_constructor_owns_deep_immutable_invariant(): @pytest.mark.parametrize("context_owned", (False, True)) def test_warmed_cache_transport_reconstructs_all_derived_defaults(context_owned): - processing_context = ProcessingContext(axis_id="A01") + processing_context = executing_context(axis_id="A01") cache = ( processing_context.runtime_source_binding_context_cache if context_owned diff --git a/tests/unit/test_runtime_stack_cache_lifetime.py b/tests/unit/test_runtime_stack_cache_lifetime.py index 2d824b4768..86b1c76ead 100644 --- a/tests/unit/test_runtime_stack_cache_lifetime.py +++ b/tests/unit/test_runtime_stack_cache_lifetime.py @@ -13,11 +13,14 @@ from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.runtime_stack_cache import RuntimeImageStackCache +from tests.context_support import ( + executing_context, +) @pytest.mark.parametrize("context_owned", [False, True]) def test_release_drops_all_cached_views_and_allows_reuse(context_owned: bool) -> None: - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") cache = ( context.runtime_image_stack_cache if context_owned else RuntimeImageStackCache() ) @@ -45,7 +48,7 @@ def test_release_drops_all_cached_views_and_allows_reuse(context_owned: bool) -> def test_context_image_cache_release_preserves_runtime_artifacts_and_observations() -> ( None ): - context = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01") output_plan = ArtifactOutputPlan( name="retained_image", path="/memory/retained.pkl", diff --git a/tests/unit/test_source_binding_workspace.py b/tests/unit/test_source_binding_workspace.py index 6918f5d07b..e83ceb1354 100644 --- a/tests/unit/test_source_binding_workspace.py +++ b/tests/unit/test_source_binding_workspace.py @@ -63,6 +63,10 @@ from polystore.filemanager import FileManager from polystore.virtual_workspace import SourcePixelRef from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + executing_context, + with_facts, +) def test_transient_axis_views_observe_each_new_projection_without_reusing_released_ids(): @@ -167,9 +171,9 @@ def observe_admission(projector, projection): assert selected.source_metadata_for(lookup) == full.source_metadata_for(lookup) assert len(full.pipeline_start_files()) == 18 from openhcs.core.context.processing_context import ProcessingContext - context = ProcessingContext(filemanager=filemanager, axis_id="A01") - context.dataset_root = workspace - context.microscope_handler = handler + context = executing_context(filemanager=filemanager, axis_id="A01") + context = with_facts(context, dataset_root=workspace) + context = with_facts(context, microscope_handler=handler) from openhcs.core import source_workspace_projection monkeypatch.setattr(source_workspace_projection, "DEFAULT_SOURCE_PROJECTION_CACHE", cache) runtime_projection = WorkspaceSourceProjections.from_context( @@ -177,9 +181,9 @@ def observe_admission(projector, projection): ).projection_or_empty() assert runtime_projection.pipeline_start_files() == selected.pipeline_start_files() assert not hasattr(context, "runtime_source_workspace_projection_cache") - other = ProcessingContext(filemanager=filemanager, axis_id="A01") - other.dataset_root = workspace - other.microscope_handler = handler + other = executing_context(filemanager=filemanager, axis_id="A01") + other = with_facts(other, dataset_root=workspace) + other = with_facts(other, microscope_handler=handler) assert context.runtime_source_workspace_projections.cache is other.runtime_source_workspace_projections.cache is cache assert other.runtime_source_workspace_projections.projection_or_empty() is selected equivalent = WorkspaceSourceProjections.from_dataset_metadata( diff --git a/tests/unit/test_stream_component_semantics.py b/tests/unit/test_stream_component_semantics.py index 3def580922..8103706e28 100644 --- a/tests/unit/test_stream_component_semantics.py +++ b/tests/unit/test_stream_component_semantics.py @@ -22,6 +22,10 @@ ) from openhcs.core.axes import AxisFamily from openhcs.domains.microscopy.axes import Microscopy +from tests.context_support import ( + compiled_context, + executing_context, +) def test_streaming_domain_spans_worker_lanes_without_mirroring_worker_identity(): @@ -38,8 +42,7 @@ def test_streaming_domain_spans_worker_lanes_without_mirroring_worker_identity() worker_slot=slot, worker_assignments=assignments.worker_assignments, ) - context = ProcessingContext() - context.bind_execution_runtime(lane) + context = compiled_context().executing(lane) assert context.execution_runtime is lane assert lane.worker_assignments is assignments.worker_assignments assert lane.owned_partition_values == tuple(assignments.worker_assignments[slot]) @@ -63,7 +66,7 @@ def test_streaming_domain_spans_worker_lanes_without_mirroring_worker_identity() def test_streaming_domain_requires_bound_execution(): with pytest.raises(RuntimeError, match="execution_runtime"): StreamExecutionAxisDomainProvider.build_for_component( - context=ProcessingContext(), + context=executing_context(), component=AxisFamily.active().partition_axis().name, metadata_roots=(), ) diff --git a/tests/unit/test_worker_start_policy.py b/tests/unit/test_worker_start_policy.py index 7c7b72039c..dea31d6666 100644 --- a/tests/unit/test_worker_start_policy.py +++ b/tests/unit/test_worker_start_policy.py @@ -10,6 +10,9 @@ WorkerStartExecutionFacts, resolve_worker_start_context, ) +from tests.context_support import ( + executing_context, +) def test_linux_cpu_server_honors_configured_fork() -> None: @@ -62,7 +65,7 @@ def test_macos_downgrades_fork_without_unsafe_override() -> None: def test_execution_facts_use_compiled_step_plan_semantics() -> None: compiled_contexts = { - "A01": ProcessingContext( + "A01": executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, @@ -88,7 +91,7 @@ def test_execution_facts_use_compiled_step_plan_semantics() -> None: def test_execution_facts_treat_numpy_contexts_as_cpu() -> None: compiled_contexts = { - "A01": ProcessingContext( + "A01": executing_context( step_plans={ 0: CompiledStepPlan( step_index=0, diff --git a/tests/unit/test_zmq_compilation.py b/tests/unit/test_zmq_compilation.py index fd2f11a398..6fcc024292 100644 --- a/tests/unit/test_zmq_compilation.py +++ b/tests/unit/test_zmq_compilation.py @@ -20,6 +20,9 @@ ZMQCompilationResult, ZMQCompileArtifactRecord, ) +from tests.context_support import ( + executing_context, +) class _StrippedStepShell: @@ -136,7 +139,7 @@ def _runtime_environment() -> CompiledRuntimeEnvironmentPlan: def _compiled_context(axis_id: str) -> ProcessingContext: - return ProcessingContext( + return executing_context( axis_id=axis_id, step_plans={ 1: CompiledStepPlan( diff --git a/tests/unit/test_zmq_execution_server.py b/tests/unit/test_zmq_execution_server.py index 051da4e032..1495599d3c 100644 --- a/tests/unit/test_zmq_execution_server.py +++ b/tests/unit/test_zmq_execution_server.py @@ -57,6 +57,10 @@ RunningProgressPhase, RunningProgressStatus, ) +from tests.context_support import ( + executing_context, + with_facts, +) def test_zmq_execution_context_seeds_saved_global_config_for_compilation() -> None: @@ -682,11 +686,11 @@ def test_compiled_source_adoption_owns_fresh_runtime_services_and_live_source_ga previous.filemanager.ensure_directory(str(tmp_path), Backend.MEMORY.value) previous.filemanager.save("stale", str(tmp_path / "old.tif"), Backend.MEMORY.value) handler = OpenHCSDatasetSource(previous.filemanager) - context = ProcessingContext(axis_id="A01", filemanager=previous.filemanager) - context.dataset_root = tmp_path - context.input_dir = tmp_path - context.microscope_handler = handler - transport = ProcessingContext(axis_id="A01") + context = executing_context(axis_id="A01", filemanager=previous.filemanager) + context = with_facts(context, dataset_root=tmp_path) + context = with_facts(context, input_dir=tmp_path) + context = with_facts(context, microscope_handler=handler) + transport = executing_context(axis_id="A01") bundle = CompiledExecutionBundle( pipeline_definition=[], runtime_contexts={"A01": context}, From ab916f2f8491a27e0db5a72f81ff5d3e4c111499 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 15:57:19 -0400 Subject: [PATCH 07/13] WIP: K7 test cleanup, dataset-step parameter names Co-Authored-By: Claude Opus 5.5 --- openhcs/core/context/processing_context.py | 8 ----- openhcs/core/orchestrator/bundle_execution.py | 34 +++++++++---------- openhcs/core/post_execute.py | 10 +++--- .../microscopy/analysis_consolidation.py | 4 +-- .../test_compiled_artifact_inspection.py | 1 - .../test_custom_source_transport.py | 1 - tests/unit/test_aligned_output_projection.py | 2 -- tests/unit/test_artifact_output_ownership.py | 4 --- tests/unit/test_artifact_plan_view.py | 2 -- tests/unit/test_cellprofiler_area_occupied.py | 1 - ...test_cellprofiler_artifact_declarations.py | 1 - ...ofiler_artifact_identity_reconstruction.py | 1 - ...ellprofiler_combineobjects_source_scope.py | 1 - ...lprofiler_generic_special_input_binding.py | 1 - ...test_cellprofiler_library_untangleworms.py | 1 - ...ellprofiler_measurement_slice_consumers.py | 1 - .../test_cellprofiler_module_execution.py | 5 --- ...ellprofiler_morphology_public_contracts.py | 1 - .../test_cellprofiler_neighbors_contract.py | 1 - .../unit/test_cellprofiler_runtime_adapter.py | 1 - ..._cellprofiler_runtime_cache_authorities.py | 1 - ...profiler_runtime_callable_introspection.py | 1 - ...profiler_runtime_input_binding_contract.py | 1 - tests/unit/test_cellprofiler_save_images.py | 1 - .../test_cellprofiler_spreadsheet_export.py | 1 - ...st_child_measurement_producer_selection.py | 1 - .../test_classification_interop_boundaries.py | 1 - tests/unit/test_compilation_session.py | 3 -- tests/unit/test_compiled_execution.py | 9 +++-- ..._completed_output_publication_lifecycle.py | 2 +- tests/unit/test_debug_runtime.py | 2 +- tests/unit/test_function_artifact_outputs.py | 2 -- .../test_function_core_executor_ownership.py | 1 - ...test_function_runtime_source_projection.py | 21 ------------ .../test_function_step_execution_scope.py | 6 ++-- tests/unit/test_function_step_transport.py | 2 -- .../test_implicit_main_flow_compilation.py | 1 - .../unit/test_invocation_contract_provider.py | 6 ---- ...nvocation_input_source_context_identity.py | 1 - .../test_loaded_pattern_group_ownership.py | 2 -- .../test_materialization_batch_outcomes.py | 1 - .../unit/test_named_checkpoint_publication.py | 2 +- .../test_opaque_main_flow_image_domain.py | 3 -- .../unit/test_output_publication_ownership.py | 2 -- .../unit/test_path_planner_materialization.py | 4 --- tests/unit/test_payload_axis_compile_gate.py | 4 --- .../test_persisted_roi_only_publication.py | 1 - .../unit/test_runtime_context_declaration.py | 1 - tests/unit/test_runtime_output_matching.py | 1 - ...test_runtime_source_resolution_snapshot.py | 1 - .../unit/test_runtime_stack_cache_lifetime.py | 1 - tests/unit/test_source_binding_workspace.py | 1 - tests/unit/test_stream_component_semantics.py | 1 - tests/unit/test_worker_start_policy.py | 1 - tests/unit/test_zmq_execution_server.py | 1 - tests/unit/test_zmq_runtime_outcome_export.py | 6 ++-- 56 files changed, 37 insertions(+), 140 deletions(-) diff --git a/openhcs/core/context/processing_context.py b/openhcs/core/context/processing_context.py index 5223ceec00..f7de4b8a5a 100644 --- a/openhcs/core/context/processing_context.py +++ b/openhcs/core/context/processing_context.py @@ -131,10 +131,6 @@ def __init__( self.pipeline_sequential_combinations = None self.current_sequential_combination = None - def inject_plan(self, step_id: int, plan: CompiledStepPlan) -> None: - """Add one step's compiled plan.""" - self.step_plans[step_id] = plan - def freeze(self) -> "CompiledContext": """Return the immutable compiled context once every fact is set.""" @@ -211,10 +207,6 @@ def __setattr__(self, name: str, value: object) -> None: CompiledContext.__setattr__(self, name, value) object.__setattr__(self, name, value) - def compiled(self) -> CompiledContext: - """Return the compiled facts this execution runs.""" - return CompiledContext(self.compiled_facts()) - @property def runtime_source_workspace_projections( self, diff --git a/openhcs/core/orchestrator/bundle_execution.py b/openhcs/core/orchestrator/bundle_execution.py index d6eb4e0c47..1d5394174c 100644 --- a/openhcs/core/orchestrator/bundle_execution.py +++ b/openhcs/core/orchestrator/bundle_execution.py @@ -506,7 +506,7 @@ def validate_dataset_scoped_contexts( def execute_dataset_scoped_steps( - compiled_contexts: Mapping[str, ExecutingContext], + executing_contexts: Mapping[str, ExecutingContext], *, progress_queue: ProgressQueue, progress_context: ProgressExecutionContext, @@ -517,15 +517,15 @@ def execute_dataset_scoped_steps( observation_cursors = { context_key: context.runtime_value_store.observation_cursor() - for context_key, context in compiled_contexts.items() + for context_key, context in executing_contexts.items() } - dataset_step_indexes = validate_dataset_scoped_contexts(compiled_contexts) - total_steps = max(next(iter(compiled_contexts.values())).step_plans) + 1 - records_by_axis = _runtime_record_snapshot(compiled_contexts) - observations_by_context = {key: [] for key in compiled_contexts} + dataset_step_indexes = validate_dataset_scoped_contexts(executing_contexts) + total_steps = max(next(iter(executing_contexts.values())).step_plans) + 1 + records_by_axis = _runtime_record_snapshot(executing_contexts) + observations_by_context = {key: [] for key in executing_contexts} for step_index in dataset_step_indexes: owner_context, owner_plan = _dataset_output_owner( - compiled_contexts, + executing_contexts, step_index, ) with ( @@ -554,7 +554,7 @@ def execute_dataset_scoped_steps( output_plan.require_single_group_key() ) batch = _dataset_artifact_batch( - compiled_contexts=compiled_contexts, + executing_contexts=executing_contexts, step_index=step_index, invocation_position=invocation_position, contract=contract, @@ -631,7 +631,7 @@ def execute_dataset_scoped_steps( ) owner_context_key = next( key - for key, context in compiled_contexts.items() + for key, context in executing_contexts.items() if context is owner_context ) observation = StepExecutionObservation.combine( @@ -670,7 +670,7 @@ def execute_dataset_scoped_steps( observations_by_context[context_key] ), ) - for context_key, context in compiled_contexts.items() + for context_key, context in executing_contexts.items() if ( records := context.runtime_value_store.observed_values_after( observation_cursors[context_key] @@ -824,13 +824,13 @@ def _validate_dataset_invocation( def _runtime_record_snapshot( - compiled_contexts: Mapping[str, ExecutingContext], + executing_contexts: Mapping[str, ExecutingContext], ) -> Mapping[str, tuple[StoredRuntimeValue, ...]]: records: OrderedDict[ str, OrderedDict[tuple[object, RuntimeArtifactLocation], StoredRuntimeValue], ] = OrderedDict() - for context in compiled_contexts.values(): + for context in executing_contexts.values(): for record in context.runtime_value_store.values(): axis_id = record.key.scope.axis_id records.setdefault(axis_id, OrderedDict()).setdefault( @@ -846,12 +846,12 @@ def _runtime_record_snapshot( def _dataset_output_owner( - compiled_contexts: Mapping[str, ExecutingContext], + executing_contexts: Mapping[str, ExecutingContext], step_index: int, ) -> tuple[ExecutingContext, CompiledStepPlan]: owners = tuple( (context, context.step_plans[step_index]) - for context in compiled_contexts.values() + for context in executing_contexts.values() if context.step_plans[step_index].owns_runtime_outputs ) if len(owners) != 1: @@ -864,7 +864,7 @@ def _dataset_output_owner( def _dataset_artifact_batch( *, - compiled_contexts: Mapping[str, ExecutingContext], + executing_contexts: Mapping[str, ExecutingContext], step_index: int, invocation_position: int, contract: CallableContract, @@ -876,12 +876,12 @@ def _dataset_artifact_batch( str, OrderedDict[tuple[object, RuntimeArtifactLocation], StoredRuntimeValue], ] = OrderedDict() - for context in compiled_contexts.values(): + for context in executing_contexts.values(): selected.setdefault(context.require_axis_id(), OrderedDict()) for input_spec in contract.artifact_inputs: selected_for_spec = 0 - for context in compiled_contexts.values(): + for context in executing_contexts.values(): axis_id = context.require_axis_id() step_plan = context.step_plans[step_index] invocation = step_plan.compiled_function_pattern.default_group.invocations[ diff --git a/openhcs/core/post_execute.py b/openhcs/core/post_execute.py index 42f3ee2e8c..bc4be28c8a 100644 --- a/openhcs/core/post_execute.py +++ b/openhcs/core/post_execute.py @@ -85,7 +85,7 @@ def combine(cls, observations: Iterable[object]) -> object | None: @abstractmethod def run( self, - compiled_contexts: Mapping[str, "ExecutingContext"], + executed_contexts: Mapping[str, "ExecutingContext"], observation: object | None, ) -> None: """Do this hook's work after every partition succeeded.""" @@ -135,7 +135,7 @@ def combine_all(observations: Iterable[HookObservations]) -> HookObservations: @staticmethod def run_all( - compiled_contexts: Mapping[str, "ExecutingContext"], + executed_contexts: Mapping[str, "ExecutingContext"], execution_results: Mapping[str, "ExecutionResult"], dataset_runtime_observation: "RuntimeExecutionObservation", ) -> None: @@ -149,7 +149,7 @@ def run_all( for context_observation in runtime_observation.contexts ) for context_observation in context_observations: - if context_observation.context_key not in compiled_contexts: + if context_observation.context_key not in executed_contexts: raise KeyError( "Runtime observation references unknown compiled context " f"{context_observation.context_key!r}." @@ -158,9 +158,9 @@ def run_all( context_observation.outputs.hook_observations for context_observation in context_observations ) - first_context = next(iter(compiled_contexts.values())) + first_context = next(iter(executed_contexts.values())) for hook in first_context.post_execute_hooks: - hook.run(compiled_contexts, observations.get(hook.hook_name)) + hook.run(executed_contexts, observations.get(hook.hook_name)) def _observations( diff --git a/openhcs/domains/microscopy/analysis_consolidation.py b/openhcs/domains/microscopy/analysis_consolidation.py index aeb06312bb..99cf53db8d 100644 --- a/openhcs/domains/microscopy/analysis_consolidation.py +++ b/openhcs/domains/microscopy/analysis_consolidation.py @@ -273,7 +273,7 @@ def combine(cls, observations): def run( self, - compiled_contexts: Mapping[str, ExecutingContext], + executed_contexts: Mapping[str, ExecutingContext], observation: RuntimeAnalysisConsolidationInputs | None, ) -> None: if not self.analysis_consolidation_config.enabled: @@ -281,7 +281,7 @@ def run( return if observation is None: return - first_context = next(iter(compiled_contexts.values())) + first_context = next(iter(executed_contexts.values())) if first_context.output_dataset_root is None: raise ValueError( "Analysis consolidation requires the compiled output plate root." diff --git a/tests/integration/test_compiled_artifact_inspection.py b/tests/integration/test_compiled_artifact_inspection.py index 33dab83f17..e4d0d9d90f 100644 --- a/tests/integration/test_compiled_artifact_inspection.py +++ b/tests/integration/test_compiled_artifact_inspection.py @@ -26,7 +26,6 @@ from openhcs.core.compiled_execution import CompiledExecutionBundle from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.component_group_scope import RuntimeExecutionAxisScope -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import ( DEFAULT_GROUP_KEY, CompiledFunctionGroup, diff --git a/tests/integration/test_custom_source_transport.py b/tests/integration/test_custom_source_transport.py index bfee92dff3..a4b8fab2df 100644 --- a/tests/integration/test_custom_source_transport.py +++ b/tests/integration/test_custom_source_transport.py @@ -455,7 +455,6 @@ def _compiled_custom_contexts(payload, runtime): from openhcs.constants.constants import Backend from openhcs.core.compiled_execution import CompiledExecutionBundle from openhcs.core.compiled_step_plan import CompiledStepPlan - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import ( compile_function_pattern, normalize_function_pattern, ) diff --git a/tests/unit/test_aligned_output_projection.py b/tests/unit/test_aligned_output_projection.py index 2e9fddb35d..bd704339a8 100644 --- a/tests/unit/test_aligned_output_projection.py +++ b/tests/unit/test_aligned_output_projection.py @@ -1,11 +1,9 @@ -from types import SimpleNamespace import numpy as np import pytest from tests.artifact_graphs import admitted_graph from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.aligned_image_payload import ( AlignedImageSliceContext, diff --git a/tests/unit/test_artifact_output_ownership.py b/tests/unit/test_artifact_output_ownership.py index 1169328f9a..a87710d7e3 100644 --- a/tests/unit/test_artifact_output_ownership.py +++ b/tests/unit/test_artifact_output_ownership.py @@ -275,7 +275,6 @@ def measure(image, *, runtime): def test_cellprofiler_row_owner_preserves_distinct_named_image_subjects(): - from openhcs.core.artifacts.relations import ArtifactSpecRelation from openhcs.processing.backends.cellprofiler.image_quality import ( MeasureImageQualityModule, ) @@ -314,7 +313,6 @@ def measure(image, *, runtime): CellProfilerRecordedArtifactOutputPolicy, ]) def test_measurement_row_owner_controls_named_object_roster_admission(policy): - from openhcs.core.artifacts.relations import ArtifactSpecRelation from openhcs.processing.backends.cellprofiler.intensity import MeasureObjectIntensityModule objects = tuple( @@ -348,7 +346,6 @@ def test_real_cellprofiler_declaration_compiles_without_a_table_wide_subject(): """The real module/provider declares the owner, not a compiler exemption.""" from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.config import GlobalPipelineConfig, PipelineConfig - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.invocation_artifacts import ArtifactDeclarationStepContext from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.core.source_bindings import NamedSourceBinding @@ -362,7 +359,6 @@ def test_real_cellprofiler_declaration_compiles_without_a_table_wide_subject(): identify_primary_objects, ) from openhcs.core.artifacts.plans import ArtifactInputPlan, ArtifactOutputPlan - from openhcs.core.artifacts.relations import ArtifactSpecRelation from openhcs.core.artifacts.kinds import ObjectLabelsArtifactType (input_binding,) = IdentifyPrimaryObjectsModule.declared_artifact_bindings( diff --git a/tests/unit/test_artifact_plan_view.py b/tests/unit/test_artifact_plan_view.py index 90baa5be9c..9efbf05d12 100644 --- a/tests/unit/test_artifact_plan_view.py +++ b/tests/unit/test_artifact_plan_view.py @@ -26,11 +26,9 @@ from openhcs.core.compiled_execution import CompiledExecutionBundle from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.debug import DebugArtifactRef, DebugCursor, DebugSnapshot from openhcs.core.function_patterns import ( DEFAULT_GROUP_KEY, diff --git a/tests/unit/test_cellprofiler_area_occupied.py b/tests/unit/test_cellprofiler_area_occupied.py index a5ae6f5892..4974777425 100644 --- a/tests/unit/test_cellprofiler_area_occupied.py +++ b/tests/unit/test_cellprofiler_area_occupied.py @@ -13,7 +13,6 @@ ) from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) diff --git a/tests/unit/test_cellprofiler_artifact_declarations.py b/tests/unit/test_cellprofiler_artifact_declarations.py index 7b3bcf5a1d..b9407a50dd 100644 --- a/tests/unit/test_cellprofiler_artifact_declarations.py +++ b/tests/unit/test_cellprofiler_artifact_declarations.py @@ -38,7 +38,6 @@ PipelineConfig, StepSourceBindingsConfig, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import ( DEFAULT_GROUP_KEY, FunctionInvocationKey, diff --git a/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py b/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py index 5230ea5de3..bd1478bc3e 100644 --- a/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py +++ b/tests/unit/test_cellprofiler_artifact_identity_reconstruction.py @@ -16,7 +16,6 @@ from openhcs.core.artifacts.kinds import ImageArtifactType, ObjectLabelsArtifactType from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( - ComponentGroupScope, UngroupedScope, ) from openhcs.core.component_set import ComponentSet diff --git a/tests/unit/test_cellprofiler_combineobjects_source_scope.py b/tests/unit/test_cellprofiler_combineobjects_source_scope.py index 3eb9fbb960..fed0801cb6 100644 --- a/tests/unit/test_cellprofiler_combineobjects_source_scope.py +++ b/tests/unit/test_cellprofiler_combineobjects_source_scope.py @@ -24,7 +24,6 @@ from openhcs.core.pipeline.function_contracts import artifact_inputs, artifact_outputs from openhcs.core.pipeline.path_planner import ( PathPlannerExecutionGroups, - ComponentGroupScope, ) from openhcs.core.progress import set_progress_queue from openhcs.core.source_bindings import NamedSourceBinding, SourceProjectionRole diff --git a/tests/unit/test_cellprofiler_generic_special_input_binding.py b/tests/unit/test_cellprofiler_generic_special_input_binding.py index 9c30764825..20f0da71b9 100644 --- a/tests/unit/test_cellprofiler_generic_special_input_binding.py +++ b/tests/unit/test_cellprofiler_generic_special_input_binding.py @@ -14,7 +14,6 @@ from openhcs.core.artifacts.kinds import ImageArtifactType, ObjectLabelsArtifactType from openhcs.core.artifacts.relations import InputStackBroadcastSourceRelation from openhcs.core.component_group_scope import ( - ComponentGroupScope, UngroupedScope, ) from openhcs.core.component_set import ComponentSet diff --git a/tests/unit/test_cellprofiler_library_untangleworms.py b/tests/unit/test_cellprofiler_library_untangleworms.py index 5de63cea10..f6e4856282 100644 --- a/tests/unit/test_cellprofiler_library_untangleworms.py +++ b/tests/unit/test_cellprofiler_library_untangleworms.py @@ -15,7 +15,6 @@ from openhcs.core.artifacts.kinds import ObjectLabelsArtifactType from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) diff --git a/tests/unit/test_cellprofiler_measurement_slice_consumers.py b/tests/unit/test_cellprofiler_measurement_slice_consumers.py index 74595a04bc..41e0aa451e 100644 --- a/tests/unit/test_cellprofiler_measurement_slice_consumers.py +++ b/tests/unit/test_cellprofiler_measurement_slice_consumers.py @@ -9,7 +9,6 @@ from openhcs.core.artifacts.kinds import MeasurementsArtifactType from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) diff --git a/tests/unit/test_cellprofiler_module_execution.py b/tests/unit/test_cellprofiler_module_execution.py index 5292195498..52c08ec72d 100644 --- a/tests/unit/test_cellprofiler_module_execution.py +++ b/tests/unit/test_cellprofiler_module_execution.py @@ -43,9 +43,7 @@ from openhcs.core.runtime_relationships import ObjectRelationshipDeclaration from openhcs.core.callable_contract import ( CallableContract, - CallableDeclaration, ImagePayloadConsumption, - attach_callable_contract_metadata, runtime_image_execution_mode, ) from openhcs.core.component_group_scope import ( @@ -161,7 +159,6 @@ ) from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.source_spatial_domain import SourceSpatialDomain -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser from openhcs.core.steps.function_runtime import ( @@ -300,7 +297,6 @@ MaskObjectsModule, closing, erode_image, - mask_objects, resize_objects_3d, ) from openhcs.processing.backends.cellprofiler.neighbors import ( @@ -377,7 +373,6 @@ ) from openhcs.core.processing_contracts import ( FlexibleContract, - ProcessingContract, Pure2DAuxiliaryOutputAggregator, Pure2DContract, Pure2DInputSlicer, diff --git a/tests/unit/test_cellprofiler_morphology_public_contracts.py b/tests/unit/test_cellprofiler_morphology_public_contracts.py index faf26b0a12..1ac90dcb34 100644 --- a/tests/unit/test_cellprofiler_morphology_public_contracts.py +++ b/tests/unit/test_cellprofiler_morphology_public_contracts.py @@ -25,7 +25,6 @@ PipelineConfig, StepSourceBindingsConfig, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_step_transport import FunctionStepTransportAuthority from openhcs.core.function_patterns import ( FunctionInvocationKey, diff --git a/tests/unit/test_cellprofiler_neighbors_contract.py b/tests/unit/test_cellprofiler_neighbors_contract.py index 82322cf749..7329664e8d 100644 --- a/tests/unit/test_cellprofiler_neighbors_contract.py +++ b/tests/unit/test_cellprofiler_neighbors_contract.py @@ -23,7 +23,6 @@ PipelineConfig, StepSourceBindingsConfig, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import ( DEFAULT_GROUP_KEY, FunctionInvocationKey, diff --git a/tests/unit/test_cellprofiler_runtime_adapter.py b/tests/unit/test_cellprofiler_runtime_adapter.py index eed6279606..c9222d0223 100644 --- a/tests/unit/test_cellprofiler_runtime_adapter.py +++ b/tests/unit/test_cellprofiler_runtime_adapter.py @@ -8,7 +8,6 @@ import numpy as np import pytest -from openhcs.constants.input_source import InputSource from openhcs.core.alias_property import AliasProperty from openhcs.core.artifacts.plans import ( ArtifactInputPlan, diff --git a/tests/unit/test_cellprofiler_runtime_cache_authorities.py b/tests/unit/test_cellprofiler_runtime_cache_authorities.py index 2fdd60e703..f4d88b649b 100644 --- a/tests/unit/test_cellprofiler_runtime_cache_authorities.py +++ b/tests/unit/test_cellprofiler_runtime_cache_authorities.py @@ -21,7 +21,6 @@ RelationshipsArtifactType, ) from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) diff --git a/tests/unit/test_cellprofiler_runtime_callable_introspection.py b/tests/unit/test_cellprofiler_runtime_callable_introspection.py index 6f6a9161e7..473127b09b 100644 --- a/tests/unit/test_cellprofiler_runtime_callable_introspection.py +++ b/tests/unit/test_cellprofiler_runtime_callable_introspection.py @@ -17,7 +17,6 @@ PipelineConfig, StepSourceBindingsConfig, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.callable_contract import CallableContract from openhcs.core.artifacts.plans import ArtifactOutputPlan from openhcs.core.artifacts.sidecars import MaterializedImageCopy diff --git a/tests/unit/test_cellprofiler_runtime_input_binding_contract.py b/tests/unit/test_cellprofiler_runtime_input_binding_contract.py index 1abce826f6..6bad72b555 100644 --- a/tests/unit/test_cellprofiler_runtime_input_binding_contract.py +++ b/tests/unit/test_cellprofiler_runtime_input_binding_contract.py @@ -22,7 +22,6 @@ ) from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) diff --git a/tests/unit/test_cellprofiler_save_images.py b/tests/unit/test_cellprofiler_save_images.py index 854791dd65..8da4a3d7ee 100644 --- a/tests/unit/test_cellprofiler_save_images.py +++ b/tests/unit/test_cellprofiler_save_images.py @@ -20,7 +20,6 @@ ) from openhcs.core.artifacts.kinds import ImageArtifactType from openhcs.core.callable_contract import CallableContract -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import ( FunctionInvocationKey, compile_function_pattern, diff --git a/tests/unit/test_cellprofiler_spreadsheet_export.py b/tests/unit/test_cellprofiler_spreadsheet_export.py index 46aa7c7ef3..a3b39cd2af 100644 --- a/tests/unit/test_cellprofiler_spreadsheet_export.py +++ b/tests/unit/test_cellprofiler_spreadsheet_export.py @@ -24,7 +24,6 @@ from openhcs.core.artifacts.plans import ArtifactInputPlan, ArtifactOutputPlan from openhcs.core.artifacts.specs import ArtifactSpec, ArtifactSpecCollection from openhcs.core.callable_contract import CallableContract, FunctionStepExecutionScope -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import FunctionInvocationKey from openhcs.core.invocation_artifacts import ArtifactDeclarationStepContext from openhcs.core.pipeline.artifact_planning import artifact_producers_for_outputs diff --git a/tests/unit/test_child_measurement_producer_selection.py b/tests/unit/test_child_measurement_producer_selection.py index 631bc68b78..1bbc93e550 100644 --- a/tests/unit/test_child_measurement_producer_selection.py +++ b/tests/unit/test_child_measurement_producer_selection.py @@ -24,7 +24,6 @@ ) from openhcs.core.pipeline.path_planner import ( PathPlannerArtifactStage, - ComponentGroupScope, ) from openhcs.core.config import EMPTY_SOURCE_BINDINGS from openhcs.processing.backends.cellprofiler.relationships import RelateObjectsModule diff --git a/tests/unit/test_classification_interop_boundaries.py b/tests/unit/test_classification_interop_boundaries.py index 5a5a9d78fd..8cd27c5277 100644 --- a/tests/unit/test_classification_interop_boundaries.py +++ b/tests/unit/test_classification_interop_boundaries.py @@ -23,7 +23,6 @@ ) from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) diff --git a/tests/unit/test_compilation_session.py b/tests/unit/test_compilation_session.py index 7b9072434a..67f6463dd7 100644 --- a/tests/unit/test_compilation_session.py +++ b/tests/unit/test_compilation_session.py @@ -63,9 +63,6 @@ ) from openhcs.core.axes import Ungrouped from openhcs.domains.microscopy.axes import Microscopy -from tests.context_support import ( - executing_context, -) def _identity(image): diff --git a/tests/unit/test_compiled_execution.py b/tests/unit/test_compiled_execution.py index 784afebcdc..e879b2d772 100644 --- a/tests/unit/test_compiled_execution.py +++ b/tests/unit/test_compiled_execution.py @@ -25,7 +25,6 @@ AnalysisConsolidationConfig, PlateMetadataConfig, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.debug import NoOpDebugExecutionPolicy from openhcs.core.measurement_row_materialization import ( MeasurementSparseColumnarRows, @@ -416,12 +415,12 @@ def test_axis_teardown_preserves_execution_error_and_attempts_every_stage( ) -> None: cleaned = [] - pixels = np.zeros((16, 16), dtype=np.uint16) - pixel_reference = weakref.ref(pixels) + held = [np.zeros((16, 16), dtype=np.uint16)] + pixel_reference = weakref.ref(held[0]) def fail_step(_pipeline, context, *_args, **_kwargs): context.runtime_image_stack_cache.store( - ("image",), memory_type="numpy", stack=pixels + ("image",), memory_type="numpy", stack=held[0] ) raise RuntimeError("step failure") @@ -469,7 +468,7 @@ def fail_reset() -> None: ) assert cleaned == [(MemoryType.CUPY, 2)] - del pixels + held.clear() assert pixel_reference() is None diff --git a/tests/unit/test_completed_output_publication_lifecycle.py b/tests/unit/test_completed_output_publication_lifecycle.py index 0c54680b36..a9276b29fe 100644 --- a/tests/unit/test_completed_output_publication_lifecycle.py +++ b/tests/unit/test_completed_output_publication_lifecycle.py @@ -4,7 +4,7 @@ from pathlib import Path from dataclasses import replace import pickle -from types import MappingProxyType, SimpleNamespace +from types import MappingProxyType import weakref import numpy as np diff --git a/tests/unit/test_debug_runtime.py b/tests/unit/test_debug_runtime.py index 2db4396218..3e6f1f8670 100644 --- a/tests/unit/test_debug_runtime.py +++ b/tests/unit/test_debug_runtime.py @@ -13,7 +13,7 @@ from openhcs.core.alias_property import AliasProperty from openhcs.constants.constants import MEMORY_TYPE_NUMPY -from openhcs.core.callable_contract import CallableContract, CallableDeclaration +from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( ComponentGroupScope, UngroupedScope, diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index eaedaa016f..a28e922427 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -10,7 +10,6 @@ from openhcs.processing.backends.cellprofiler.crop import CropMask from openhcs.constants.constants import MEMORY_TYPE_NUMPY from openhcs.core.artifacts.kinds import ( - ArtifactType, ImageArtifactType, ObjectLabelsArtifactType, MeasurementsArtifactType, @@ -1773,7 +1772,6 @@ def rgb_output(image): artifact_kind=ImageArtifactType.value, ), ) - from openhcs.core.aligned_image_payload import ImagePayloadStackComposition stack_payload = (scalar_output).copied(memory_type=MEMORY_TYPE_NUMPY, device_id=None,) diff --git a/tests/unit/test_function_core_executor_ownership.py b/tests/unit/test_function_core_executor_ownership.py index 4012c31194..6a74e4c41e 100644 --- a/tests/unit/test_function_core_executor_ownership.py +++ b/tests/unit/test_function_core_executor_ownership.py @@ -13,7 +13,6 @@ from openhcs.core.artifacts.kinds import MetadataArtifactType from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.component_group_scope import ( - ComponentGroupScope, UngroupedScope, ) from openhcs.core.function_patterns import ( diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index be92abddab..c6508bab89 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -2167,7 +2167,6 @@ def test_first_step_prepares_raw_source_anchors_under_semantic_binding_groups( from openhcs.core.progress import set_progress_queue from openhcs.core.config import LazyStepSourceBindingsConfig from openhcs.core.steps.function_runtime import ( - PatternGroupExecutionScope, PatternGroupExecutionRequest, ) from openhcs.core.steps.function_step import FunctionStep @@ -2400,7 +2399,6 @@ def test_grouped_runtime_adapter_receives_component_selected_source_bindings() - RuntimePlaneProjection, ) from openhcs.core.steps.function_runtime import ( - PatternGroupData, FunctionCoreExecutor, ) @@ -2476,9 +2474,6 @@ def test_runtime_invocation_uses_only_active_source_bound_main_flow_edges( ) -> None: from openhcs.core.artifacts.payloads import NoMainFlowOutput from openhcs.core.steps import function_runtime - from openhcs.core.steps.function_runtime import ( - PatternGroupData, - ) source_binding_plan = CompiledSourceBindingPlan( bindings=tuple( @@ -2618,9 +2613,7 @@ def execute(self, *, debug_sink=None): def test_source_roster_selection_cannot_replace_missing_stored_primary_epoch() -> None: from openhcs.core.artifacts.input_projection import ArtifactInputProjectionPlan from openhcs.core.steps.function_runtime import ( - PatternGroupExecutionScope, FunctionCoreExecutor, - PatternGroupData, ) bindings = CompiledSourceBindingPlan( @@ -2701,9 +2694,6 @@ def test_runtime_chain_skips_adapter_invocation_without_component_outputs( ) -> None: from openhcs.core.function_patterns import CompiledFunctionGroup from openhcs.core.steps import function_runtime - from openhcs.core.steps.function_runtime import ( - PatternGroupData, - ) first_spec = ArtifactSpec.output("FirstLabels", ObjectLabelsArtifactType) second_spec = ArtifactSpec.output("SecondLabels", ObjectLabelsArtifactType) @@ -3362,7 +3352,6 @@ def test_pipeline_start_main_flow_survives_prior_producer_image_input( def test_runtime_plane_count_comes_from_loaded_slices_not_dispatch_groups() -> None: from openhcs.core.steps.function_runtime import ( - PatternGroupData, PatternGroupExecutionRequest, ) @@ -3550,8 +3539,6 @@ def record_mixed_outputs(image, *, runtime): def test_component_output_selection_keeps_distinct_axes_with_equal_keys() -> None: from openhcs.core.steps.function_runtime import ( PatternGroupExecutionScope, - FunctionCoreExecutor, - PatternGroupData, ) channel_1 = ArtifactOutputPlan( @@ -3584,11 +3571,6 @@ def test_component_output_selection_keeps_distinct_axes_with_equal_keys() -> Non def test_component_artifact_plans_reject_malformed_exact_plan_maps() -> None: - from openhcs.core.steps.function_runtime import ( - PatternGroupExecutionScope, - FunctionCoreExecutor, - PatternGroupData, - ) input_plan = ArtifactInputPlan( name="InputImage", @@ -3786,7 +3768,6 @@ def matched_image_set_from_plan(*, bindings, **_kwargs): def test_alias_only_workspace_filter_excludes_unselected_source_and_orders_stack( monkeypatch: pytest.MonkeyPatch, ) -> None: - from openhcs.core.steps import function_runtime declarations = ( NamedSourceBinding(alias="Hoechst"), @@ -3867,7 +3848,6 @@ def test_unbound_workspace_source_keeps_filename_component_provenance( ) -> None: """Ordinary workspace sources retain fixed coordinates without bindings.""" - from openhcs.core.steps import function_runtime virtual_path = "A01_s002_w1_z003_t004.tif" full_virtual_path = str(tmp_path / virtual_path) @@ -5619,7 +5599,6 @@ def test_save_outputs_positional_lowering_preserves_explicit_payload_identity( from openhcs.core.steps.function_runtime import PatternGroupExecutionRequest from openhcs.core.runtime_stack_cache import RuntimeImageStackCache from openhcs.core.compiled_step_plan import CompiledStepPlan - from openhcs.core.component_group_scope import ComponentGroupScope from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.aligned_image_payload import AlignedImageStack diff --git a/tests/unit/test_function_step_execution_scope.py b/tests/unit/test_function_step_execution_scope.py index 1cff072025..f02c81e106 100644 --- a/tests/unit/test_function_step_execution_scope.py +++ b/tests/unit/test_function_step_execution_scope.py @@ -607,7 +607,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): MeasurementsArtifactType, } batch = _dataset_artifact_batch( - compiled_contexts={"A01": context}, + executing_contexts={"A01": context}, step_index=1, invocation_position=0, contract=CallableContract.from_callable(export), @@ -716,7 +716,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): retained_store.merge_observed_values(retained) batches = tuple( _dataset_artifact_batch( - compiled_contexts={"A01": context}, + executing_contexts={"A01": context}, step_index=1, invocation_position=0, contract=CallableContract.from_callable(export), @@ -765,7 +765,7 @@ def export(*, artifact_batch: RuntimeArtifactBatch): assert MergeDatasetInputRecords.retain_records((), context) == () with pytest.raises(ValueError, match="missing required input"): _dataset_artifact_batch( - compiled_contexts={"A01": context}, + executing_contexts={"A01": context}, step_index=1, invocation_position=0, contract=CallableContract.from_callable(export), diff --git a/tests/unit/test_function_step_transport.py b/tests/unit/test_function_step_transport.py index 45b1db6891..ec40b68158 100644 --- a/tests/unit/test_function_step_transport.py +++ b/tests/unit/test_function_step_transport.py @@ -45,7 +45,6 @@ from openhcs.domains.microscopy.axes import Microscopy from tests.context_support import ( compiled_context, - executing_context, ) @@ -398,7 +397,6 @@ def test_bundle_transport_preserves_prepared_runtime_contract_identity() -> None CompiledRuntimeEnvironmentPlan, ) from openhcs.core.compiled_step_plan import CompiledStepPlan - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import compile_function_pattern, normalize_function_pattern from openhcs.core.orchestrator.execution_result import ( RuntimeExecutionTransportSerialization, diff --git a/tests/unit/test_implicit_main_flow_compilation.py b/tests/unit/test_implicit_main_flow_compilation.py index 75d1203de9..f58c11d00b 100644 --- a/tests/unit/test_implicit_main_flow_compilation.py +++ b/tests/unit/test_implicit_main_flow_compilation.py @@ -15,7 +15,6 @@ PipelineConfig, ) from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator -from openhcs.core.pipeline.path_planner import ComponentGroupScope from openhcs.core.progress import set_progress_queue from openhcs.core.step_dependencies import StepInputDependency from openhcs.core.steps.function_step import FunctionStep diff --git a/tests/unit/test_invocation_contract_provider.py b/tests/unit/test_invocation_contract_provider.py index 63c63ac3e0..4753d591de 100644 --- a/tests/unit/test_invocation_contract_provider.py +++ b/tests/unit/test_invocation_contract_provider.py @@ -667,7 +667,6 @@ def test_cellprofiler_provider_reconstructs_exact_contract_from_public_step() -> LazyProcessingConfig, PipelineConfig, ) - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.core.source_bindings import NamedSourceBinding from openhcs.core.config import StepSourceBindingsConfig @@ -759,7 +758,6 @@ def test_cellprofiler_provider_advances_native_artifacts_through_generic_graph() ): from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.config import GlobalPipelineConfig, PipelineConfig - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.core.steps.function_step import FunctionStep from openhcs.interop.cellprofiler.compile_time_contracts import ( @@ -827,7 +825,6 @@ def test_cellprofiler_provider_advances_native_artifacts_through_generic_graph() def test_cellprofiler_provider_leaves_native_same_name_callable_unclaimed() -> None: from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.config import GlobalPipelineConfig, PipelineConfig - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.core.steps.function_step import FunctionStep from openhcs.interop.cellprofiler.compile_time_contracts import ( @@ -1116,7 +1113,6 @@ def test_cellprofiler_provider_rejects_under_specified_one_image_align() -> None LazyProcessingConfig, PipelineConfig, ) - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.core.source_bindings import NamedSourceBinding from openhcs.core.config import StepSourceBindingsConfig @@ -1235,7 +1231,6 @@ def test_calculate_math_provider_keeps_object_identity_and_output_name_public() ) from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.config import GlobalPipelineConfig, PipelineConfig - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.core.source_bindings import NamedSourceBinding, SourceProjectionRole from openhcs.core.config import StepSourceBindingsConfig @@ -1357,7 +1352,6 @@ def test_native_unnamed_main_flow_remains_a_canonical_contract_input() -> None: LazyProcessingConfig, PipelineConfig, ) - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.core.steps.function_step import FunctionStep from openhcs.interop.cellprofiler.compile_time_contracts import ( diff --git a/tests/unit/test_invocation_input_source_context_identity.py b/tests/unit/test_invocation_input_source_context_identity.py index e4c2c83640..1ef9d9860c 100644 --- a/tests/unit/test_invocation_input_source_context_identity.py +++ b/tests/unit/test_invocation_input_source_context_identity.py @@ -12,7 +12,6 @@ from openhcs.core.artifacts.kinds import ImageArtifactType from openhcs.core.callable_contract import CallableContract from openhcs.core.component_group_scope import ( - ComponentGroupScope, RuntimeExecutionAxisScope, UngroupedScope, ) diff --git a/tests/unit/test_loaded_pattern_group_ownership.py b/tests/unit/test_loaded_pattern_group_ownership.py index 57e22fd428..08b9178895 100644 --- a/tests/unit/test_loaded_pattern_group_ownership.py +++ b/tests/unit/test_loaded_pattern_group_ownership.py @@ -11,11 +11,9 @@ from tests.artifact_graphs import admitted_graph from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.component_group_scope import ( - ComponentGroupScope, DynamicGroupScope, UngroupedScope, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.source_bindings import CompiledSourceBindingPlan diff --git a/tests/unit/test_materialization_batch_outcomes.py b/tests/unit/test_materialization_batch_outcomes.py index 1b251698e2..9c5a443752 100644 --- a/tests/unit/test_materialization_batch_outcomes.py +++ b/tests/unit/test_materialization_batch_outcomes.py @@ -13,7 +13,6 @@ from polystore.memory import MemoryStorageBackend from tests.artifact_graphs import admitted_graph -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.debug import NoOpDebugExecutionPolicy from openhcs.core.function_patterns import compile_function_pattern diff --git a/tests/unit/test_named_checkpoint_publication.py b/tests/unit/test_named_checkpoint_publication.py index bfbb747d75..9566b6d3f5 100644 --- a/tests/unit/test_named_checkpoint_publication.py +++ b/tests/unit/test_named_checkpoint_publication.py @@ -11,7 +11,7 @@ import pytest import tifffile -from test_artifact_publication_journey import _plate, progress_events +from test_artifact_publication_journey import _plate from openhcs.core.artifacts.sidecars import QaCheckpoint from openhcs.core.artifacts.kinds import ImageArtifactType from openhcs.core.artifacts.specs import ( diff --git a/tests/unit/test_opaque_main_flow_image_domain.py b/tests/unit/test_opaque_main_flow_image_domain.py index 266865185a..398442a0d5 100644 --- a/tests/unit/test_opaque_main_flow_image_domain.py +++ b/tests/unit/test_opaque_main_flow_image_domain.py @@ -10,7 +10,6 @@ from openhcs.constants.constants import MEMORY_TYPE_NUMPY from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.component_group_scope import ( - ComponentGroupScope, UngroupedScope, ) from openhcs.core.function_patterns import MainFlowInputProjection, compile_function_pattern @@ -20,9 +19,7 @@ from openhcs.core.source_spatial_domain import SourceSpatialDomain from openhcs.core.steps.function_runtime import PatternGroupExecutionRequest from openhcs.core.aligned_image_payload import AlignedImageStack, ImageOutputBundle, AlignedImageSliceContext, ImagePayloadStackComposition -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.steps.function_output_manifest import step_output_manifest, StepOutputManifestStore -from openhcs.core.source_bindings import CompiledSourceBindingPlan from openhcs.core.step_dependencies import StepInputDependency from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser from openhcs.core.source_image_provenance import SourceImageProvenancePlanes diff --git a/tests/unit/test_output_publication_ownership.py b/tests/unit/test_output_publication_ownership.py index 1b9f4f60c2..f2608abd2a 100644 --- a/tests/unit/test_output_publication_ownership.py +++ b/tests/unit/test_output_publication_ownership.py @@ -17,10 +17,8 @@ ) from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.component_group_scope import ( - ComponentGroupScope, UngroupedScope, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.steps.function_output_manifest import step_output_manifest diff --git a/tests/unit/test_path_planner_materialization.py b/tests/unit/test_path_planner_materialization.py index d79a9b28ff..bf39dfe5a8 100644 --- a/tests/unit/test_path_planner_materialization.py +++ b/tests/unit/test_path_planner_materialization.py @@ -43,7 +43,6 @@ from openhcs.constants.constants import MEMORY_TYPE_NUMPY from openhcs.core.memory import numpy from openhcs.core.runtime_image_values import ImagePayloadMetadata -from openhcs.core.runtime_plane_projection import RuntimePlaneProjection from openhcs.core.runtime_stores import RuntimeValueStore from openhcs.core.runtime_artifact_values import RuntimeValue from openhcs.core.config import ( @@ -92,7 +91,6 @@ PathPlannerComponentScopes, PathPlannerArtifactStage, PathPlannerExecutionGroups, - ComponentGroupScope, PathPlannerMaterializationStage, PathPlannerPathAuthority, PathPlannerStepAssemblyStage, @@ -113,7 +111,6 @@ ) from openhcs.core.step_dependencies import StepInputDependency from openhcs.core.step_dependencies import StepInputDependencyKind -from openhcs.core.steps.abstract import AbstractStep from openhcs.core.steps.function_step import FunctionStep from openhcs.core.steps.function_runtime import ( PatternGroupExecutionScope, @@ -1167,7 +1164,6 @@ def test_implicit_native_main_flow_provenance_drives_artifact_owned_scope(): LazyProcessingConfig, PipelineConfig, ) - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_patterns import normalize_function_pattern from openhcs.core.pipeline.compilation_session import CompilationSession from openhcs.interop.cellprofiler.compile_time_contracts import ( diff --git a/tests/unit/test_payload_axis_compile_gate.py b/tests/unit/test_payload_axis_compile_gate.py index 2e66fd2583..72cebb14d3 100644 --- a/tests/unit/test_payload_axis_compile_gate.py +++ b/tests/unit/test_payload_axis_compile_gate.py @@ -19,7 +19,6 @@ from openhcs.core.component_group_scope import ComponentGroupScope from openhcs.core.context.processing_context import ( CompilingContext, - ProcessingContext, ) from openhcs.core.function_patterns import ( MainFlowInputProjection, @@ -44,9 +43,6 @@ from openhcs.processing.backends.cellprofiler.color import color_to_gray from openhcs.processing.backends.cellprofiler.crop import crop from openhcs.domains.microscopy.axes import Microscopy -from tests.context_support import ( - executing_context, -) class _CarrierTestSession(SimpleNamespace): diff --git a/tests/unit/test_persisted_roi_only_publication.py b/tests/unit/test_persisted_roi_only_publication.py index 81b46d3bec..2700176a6f 100644 --- a/tests/unit/test_persisted_roi_only_publication.py +++ b/tests/unit/test_persisted_roi_only_publication.py @@ -6,7 +6,6 @@ from polystore.disk import DiskStorageBackend from polystore.filemanager import FileManager -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser import json diff --git a/tests/unit/test_runtime_context_declaration.py b/tests/unit/test_runtime_context_declaration.py index 3409a63e94..cd1de0c5fa 100644 --- a/tests/unit/test_runtime_context_declaration.py +++ b/tests/unit/test_runtime_context_declaration.py @@ -15,7 +15,6 @@ prepare_processing_callable, ) from openhcs.core.compiled_step_plan import CompiledStepPlan -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.function_contract_metadata import FunctionContractAttribute from openhcs.core.function_patterns import compile_function_pattern from openhcs.core.function_reference import FunctionReferenceTransportAuthority diff --git a/tests/unit/test_runtime_output_matching.py b/tests/unit/test_runtime_output_matching.py index cf2ffa079b..a9cf8a7bad 100644 --- a/tests/unit/test_runtime_output_matching.py +++ b/tests/unit/test_runtime_output_matching.py @@ -16,7 +16,6 @@ ) from openhcs.core.callable_contract import ( CallableContract, - CallableDeclaration, FunctionStepExecutionScope, ) from openhcs.core.runtime_image_values import ImagePayloadMetadata diff --git a/tests/unit/test_runtime_source_resolution_snapshot.py b/tests/unit/test_runtime_source_resolution_snapshot.py index 709dc3a946..f96b794440 100644 --- a/tests/unit/test_runtime_source_resolution_snapshot.py +++ b/tests/unit/test_runtime_source_resolution_snapshot.py @@ -6,7 +6,6 @@ import pytest from polystore.virtual_workspace import SourcePixelRef -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.runtime_source_binding_cache import RuntimeSourceBindingContextCache from openhcs.core.source_binding_selection import ( DeclaredSourceMetadataRecord, diff --git a/tests/unit/test_runtime_stack_cache_lifetime.py b/tests/unit/test_runtime_stack_cache_lifetime.py index 86b1c76ead..6a38b4fbb6 100644 --- a/tests/unit/test_runtime_stack_cache_lifetime.py +++ b/tests/unit/test_runtime_stack_cache_lifetime.py @@ -8,7 +8,6 @@ from openhcs.core.artifacts.plans import ArtifactOutputPlan from openhcs.core.artifacts.kinds import ImageArtifactType -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.runtime_artifact_values import RuntimeValue from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis diff --git a/tests/unit/test_source_binding_workspace.py b/tests/unit/test_source_binding_workspace.py index e83ceb1354..2706746dc4 100644 --- a/tests/unit/test_source_binding_workspace.py +++ b/tests/unit/test_source_binding_workspace.py @@ -170,7 +170,6 @@ def observe_admission(projector, projection): assert selected.require_source_projection_for(lookup) is full.require_source_projection_for(lookup) assert selected.source_metadata_for(lookup) == full.source_metadata_for(lookup) assert len(full.pipeline_start_files()) == 18 - from openhcs.core.context.processing_context import ProcessingContext context = executing_context(filemanager=filemanager, axis_id="A01") context = with_facts(context, dataset_root=workspace) context = with_facts(context, microscope_handler=handler) diff --git a/tests/unit/test_stream_component_semantics.py b/tests/unit/test_stream_component_semantics.py index 8103706e28..1900c04cfb 100644 --- a/tests/unit/test_stream_component_semantics.py +++ b/tests/unit/test_stream_component_semantics.py @@ -14,7 +14,6 @@ StreamSourceComponentMetadataItems, StreamViewerComponentMetadataProjector, ) -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.debug import NoOpDebugExecutionPolicy from openhcs.core.orchestrator.worker_lanes import ( WorkerAssignmentPlan, diff --git a/tests/unit/test_worker_start_policy.py b/tests/unit/test_worker_start_policy.py index dea31d6666..6de9fd33b7 100644 --- a/tests/unit/test_worker_start_policy.py +++ b/tests/unit/test_worker_start_policy.py @@ -4,7 +4,6 @@ from openhcs.core.compiled_step_plan import CompiledStepPlan from openhcs.core.config import MultiprocessingStartMethod -from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.worker_start_policy import ( WorkerStartPlatform, WorkerStartExecutionFacts, diff --git a/tests/unit/test_zmq_execution_server.py b/tests/unit/test_zmq_execution_server.py index 1495599d3c..0ee0daf60b 100644 --- a/tests/unit/test_zmq_execution_server.py +++ b/tests/unit/test_zmq_execution_server.py @@ -652,7 +652,6 @@ def test_compiled_source_adoption_owns_fresh_runtime_services_and_live_source_ga CompiledExecutionBundle, CompiledRuntimeEnvironmentPlan, ) - from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.orchestrator.cancellation import ExecutionCancelledError from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.dataset_sources.openhcs_format import OpenHCSDatasetSource diff --git a/tests/unit/test_zmq_runtime_outcome_export.py b/tests/unit/test_zmq_runtime_outcome_export.py index 7b9faf19e7..2cbe9da8c4 100644 --- a/tests/unit/test_zmq_runtime_outcome_export.py +++ b/tests/unit/test_zmq_runtime_outcome_export.py @@ -107,7 +107,7 @@ def test_outcome_export_rejects_missing_and_uncompiled_axes(tmp_path: Path) -> N def test_value_export_rejects_an_uncompiled_execution_axis() -> None: exported = ZMQRuntimeExecutionObservationExport.from_execution( - compiled_contexts={}, + executed_contexts={}, execution_results={"A01": ExecutionSucceeded("A01")}, output_roots=(), runtime_observations=(), @@ -121,7 +121,7 @@ def test_runtime_observation_export_never_overwrites_existing_file( ) -> None: observation_path = tmp_path / "observation.pkl.gz" ZMQRuntimeExecutionObservationExport.from_execution( - compiled_contexts={}, + executed_contexts={}, execution_results={}, output_roots=(), execution_id="first-job", @@ -130,7 +130,7 @@ def test_runtime_observation_export_never_overwrites_existing_file( retained_bytes = observation_path.read_bytes() with pytest.raises(FileExistsError): ZMQRuntimeExecutionObservationExport.from_execution( - compiled_contexts={}, + executed_contexts={}, execution_results={}, output_roots=(), execution_id="other-job", From 9d916e08799d0508c5b69e861d426f7e4aaf9947 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 16:20:11 -0400 Subject: [PATCH 08/13] WIP: K7 integration tests read executed contexts; receipt as built Co-Authored-By: Claude Opus 5.5 --- .../K7-execution-protocol.md | 56 ++++++++++++++++--- .../test_chained_callable_lineage_journey.py | 2 +- .../test_custom_source_transport.py | 5 +- .../test_measurement_declaration_journey.py | 8 ++- ...rimary_segmentation_diagnostics_journey.py | 2 +- 5 files changed, 57 insertions(+), 16 deletions(-) diff --git a/docs/refactor/consolidation-20261009/K7-execution-protocol.md b/docs/refactor/consolidation-20261009/K7-execution-protocol.md index 7a460731d1..df0cbfb3f9 100644 --- a/docs/refactor/consolidation-20261009/K7-execution-protocol.md +++ b/docs/refactor/consolidation-20261009/K7-execution-protocol.md @@ -19,21 +19,59 @@ Re-measured at this head. G7 replaced part of K7 (see Already done); the other t 3. **`ProcessingContext` mixes compile-time and execution-time state** (`core/context/processing_context.py:64`, 324 lines, 258 references in 45 production files). One mutable object carries the compiler's facts (`step_plans`, `axis_id`, `filemanager`, sequential combinations, visualizers, axis filters) and the executor's (`runtime_value_store` and four runtime caches, `completed_step_outputs`, `execution_runtime`, `debug_event_sink`, the per-step value scope). The phase is a flag: `_is_frozen`, checked by `is_frozen()` in 6 compiler stages (`compiler.py:252,536,660,738,775,800`) and in `worker_execution.py:1473`. The compiled context is pickled to workers with empty runtime stores, executed in place, and its stores cleared afterwards (`worker_execution.py:1357`). `compiler.py:227` sets `context.visualizer_config`, an attribute nothing declares or reads. 4. **`ExecutionResult` is a longhand union** (`orchestrator/execution_result.py:190`): an `ExecutionStatus` enum plus `failed_combination` and `error_message` Optionals meaningful only for some statuses, three `is_*` predicates, and three constructors that restate the field defaults. `RuntimeObservationMode` (same file) is an enum whose members' behaviour is written as `if self is …` branches (rule 1a). -## Target - -- **Scope variants.** `ComponentGroupScope` is an ABC with `UngroupedScope`, and `GroupedScope` (holds `component`) with `DynamicGroupScope` and `StaticGroupScope` (holds `keys`). Each variant owns `runtime_keys`, `contains_runtime_key`, `contains_scope`, `resolve_runtime_key`, `select_runtime_key`, `excludes`, `output_lineage_scope` and the other questions consumers asked through predicates. `ComponentGroupScope.of(keys, component)` is the one constructor from raw values. The predicates and `from_raw` are deleted. -- **Plans hold a scope.** `ArtifactPlan.group_scope` is a field; `group_keys`/`group_component` and every unpack/re-pack are deleted. -- **Two compositions.** `RuntimeValue.compose(values)` and `RuntimeValue.compose_groups(values, producer_scope: GroupedScope)`; kinds override `compose_runtime_groups` where reconstruction differs. The `producer_group_scope` parameter is deleted. -- **Phase types.** `ProcessingContext` is the base carrying compile facts. `CompilingContext` is the only mutable one (`inject_plan`, `freeze() -> CompiledContext`); `CompiledContext` is immutable and is what bundles and workers carry; `ExecutingContext(CompiledContext)` adds the runtime stores, caches, debug sink, lane runtime and completed outputs, and is built per execution by `CompiledContext.executing(...)`. `_is_frozen`, `is_frozen()`, the six compiler checks and `visualizer_config` are deleted. -- **One result family.** `ExecutionResult` ABC with `ExecutionSucceeded`, `ExecutionFailed` (owns `error_message`, `failed_combination`) and `ExecutionCancelled` (owns `error_message`); `ExecutionStatus` and the `is_*` predicates are deleted. `RuntimeObservationMode` becomes a family whose members own `retain_records`. +## Target (as built) + +- **Scope variants** (`core/component_group_scope.py`). `ComponentGroupScope` is an ABC: `UngroupedScope`; `GroupedScope` (holds `component`) with `DynamicGroupScope`, `StaticGroupScope` (two or more keys) and `SingleKeyScope` (exactly one). Each variant owns `runtime_keys`, `candidate_keys`, `contains_runtime_key`, `contains_scope`, `resolve_runtime_key`, `select_runtime_key`, `single_key`, `excludes`, `selectable_from`, `owned_by`, `admits_record_key`, `value_for_plane`, `may_supply_key`, `shared_invocation_scope` and `output_lineage_scope`. `as_grouped()` yields the typed grouped scope (a non-`None` component) where a caller acts only on grouped scopes; `has_compiled_keys` and `names_several_keys` are declared per variant. `ComponentGroupScope.of(keys, component)` is the one reader of the raw spelling. Deleted: the six predicates, `from_raw`, `ungrouped()`, `dynamic()`, `PathPlannerGroupScope` (its union moved to `ComponentGroupScope.union_compatible`; `output_groups_for` and `missing_from` had no caller), and the seven "lost its component" checks the types now make impossible. +- **Plans hold a scope.** `ArtifactPlan.group_scope` is a field. Deleted: `group_keys`, `group_component`, `group_scope()`, `ArtifactInputPlan.producer_group_scope()`, `single_group_key`, `has_dynamic_group_scope`; `relation_scope(component)` replaces `PathPlannerGroupScope.relation_scope_from_plan`. +- **Two compositions.** `RuntimeValue.compose(values)` and `RuntimeValue.compose_groups(values, producer_scope: GroupedScope)`; kinds implement `compose_runtime_values` and, where rebuilding the group axis differs (image, object labels, object lineage), `compose_runtime_groups`. The `producer_group_scope` parameter is deleted. +- **Phase types** (`core/context/processing_context.py`). `ProcessingContext` is the base (`require_axis_id`, `require_parameter_name`). `CompilingContext` is the only mutable phase; `freeze()` returns a `CompiledContext`, which rejects every attribute assignment and is what bundles, transport and workers carry. `CompiledContext.executing(lane)` starts one execution: `ExecutingContext` adds the runtime value store, the four runtime caches, the debug sink, the lane runtime, completed outputs and the per-step value scope, and still rejects assignment to compiled facts. A worker lane builds one per compiled context it executes; the parent builds one per compiled context per `execute_compiled_bundle` call, merges worker observations into it, runs dataset-scoped steps and post-execute hooks on it, and returns them as `BundleExecutionResults.executed_contexts` (the ZMQ observation export reads them there, not from the bundle). Deleted: `_is_frozen`, `is_frozen()`, the six compiler checks and the worker check, `bind_execution_runtime`, `reset_completed_step_outputs`, `inject_plan`, `visualizer_config` (always `None`) and the `core.context` re-export. +- **One result family** (`core/orchestrator/execution_result.py`). `ExecutionResult` ABC with `ExecutionSucceeded`, and `ExecutionStopped` (owns `error_message`, `failed_combination`, `progress_phase`, `progress_status`, `at_combination`) with `ExecutionFailed` and `ExecutionCancelled` (`stops_lane`). Deleted: `ExecutionStatus`, `is_error`, `is_cancelled`, the three constructors, and `ZMQExecutionAxisOutcome` (the outcome export stores `result.without_runtime_observation()`; schema version 5). `RuntimeObservationMode` is a family: `OmitRuntimeRecords`, `MergeAllRuntimeRecords`, `MergeDatasetInputRecords` (E1's names), each owning `retain_records`. + +Production (Python under `openhcs/`): see the PR status line. The net is small because three families replace three encodings one for one; the deletions are the predicates, re-packs and checks, the additions are the variant classes. + +## Deletion certificates + +| Deleted | Certificate | +|---|---| +| `is_ungrouped`/`is_grouped`/`is_dynamic`/`is_static`/`has_single_static_key`/`require_single_static_key`, `from_raw`, `ungrouped()`, `dynamic()` | **derived** (the variant classes and `ComponentGroupScope.of`) | +| "lost its component" `RuntimeError`s (7), `ArtifactInputProjectionPlan` re-normalisation | **derived** (`GroupedScope.component` is required by construction) | +| `ArtifactPlan.group_keys`/`group_component`, `group_scope()`, `producer_group_scope()`, `single_group_key`, `has_dynamic_group_scope` | **derived** (the `group_scope` field) | +| `PathPlannerGroupScope.output_groups_for`, `.missing_from` | **unused** | +| `producer_group_scope` parameter of `compose`/`compose_runtime_values` | **derived** (`compose_groups`/`compose_runtime_groups`) | +| `ProcessingContext._is_frozen`, `is_frozen()`, 6 compiler checks, the worker "not frozen" check | **derived** (`CompilingContext`/`CompiledContext` types) | +| `bind_execution_runtime`, `reset_completed_step_outputs` | **derived** (`CompiledContext.executing(lane)` builds a fresh execution) | +| `CompilingContext.inject_plan` | **unused** | +| `context.visualizer_config` and the `vis_config` argument at the one call | **derived** (it was always `None`, the parameter's default) | +| `ExecutionStatus`, `is_error`, `is_cancelled`, `ExecutionResult.success/error/cancelled` | **derived** (the result subclasses) | +| `ZMQExecutionAxisOutcome` | **derived** (`ExecutionResult.without_runtime_observation`) | +| `RuntimeObservationMode.from_parent_requirement` | **unused** | +| `openhcs.core.context.ProcessingContext` re-export | **contract** (no re-exports) | + +**Tests-only** (behaviour the interface admits that no test exercises): +- `ComponentGroupScope.of` removes duplicate keys and reads an empty key tuple as "no named keys"; `from_raw` kept duplicates and rejected an empty tuple. No producer of duplicate or empty keys is tested. +- `RuntimeArtifactInput.records` no longer special-cases an ungrouped producer; the general path selects the same record. A relation that declares a complete-producer selection on an ungrouped producer now runs the declaration check it previously skipped. +- `ArtifactInputProjectionPlan.validate_storage_plan` raises one message for every out-of-scope selection (three before). +- Runtime caches (`runtime_image_stack_cache`, pattern-discovery, source-binding, output-identity) now live for one execution of a compiled context. Before, in-process lanes and repeated executions of one bundle shared them through the compiled context. Reuse across executions of one bundle is untested; correctness does not depend on it, timing might. +- Thread and inline lanes no longer execute on the parent's context object; they get their own `ExecutingContext` like process lanes, and only returned observations reach the parent. Code that read a parent context's store mid-execution from another thread is untested (none found). +- A cancelled or failed axis reports `type(result).__name__` in `require_success` and export messages (`ExecutionFailed: …`), not `error: …`. + +## Reassigned / found outside this surface + +| Item | Evidence | Belongs to | +|---|---|---| +| `NamedSourceBinding.input_plan(group_keys=, group_component=)` has no caller and names the deleted plan fields | `core/source_bindings.py:1298` | E1 (owns the file in #1203): delete | +| Annotations `ProcessingContext` on functions that read `execution_runtime` | `processing/materialization/core.py`, `path_scopes.py` | K8: annotate `ExecutingContext` | +| `RuntimeExecutionAxisScope(component=None, value=None)` packs "no group coordinate" the same way the scope did | `core/component_group_scope.py` | not assigned; for the session lead | +| E1 (#1203) also converts `ExecutionStatus` and `RuntimeObservationMode`; this surface uses E1's mode names and replaces the status family with result subclasses | `execution_result.py` | whichever merges second resolves in favour of the result family | +| `PipelineOrchestrator.execute_compiled_bundle(runtime_observation_mode=None)` derives the mode when `None` | `orchestrator.py:765` | G7 follow-up, not assigned | ## Guards -`tests/unit/test_k7_execution_protocol_guards.py`: no production read of the deleted scope predicates or `from_raw`; no `group_keys`/`group_component` plan fields; no `producer_group_scope` parameter; no `is_frozen`/`_is_frozen`; no `ExecutionStatus` in `execution_result.py`; the scope, context and result families are exactly the declared members. +`tests/unit/test_k7_execution_protocol_guards.py`: none of the deleted names exists in `openhcs/`; the scope, context-phase, result and observation-mode families are exactly the declared members; `ArtifactPlan` has `group_scope` and neither old field; each scope variant reads back from its raw spelling and pickles; each context phase carries only its own state and compiled facts cannot be reassigned; stopped results own their message and combination. ## Tests -Touched tests take the new constructors and types. Run: the guards; the unit test files that import the changed modules (one process each); official30 parity (30/30). +Tests take the new constructors (`tests/context_support.py` builds phase contexts without the compiler). Deleted assertions were on structure only: that a transport context shares the runtime store, that the passed context object was the one executed and cleared. Run: the guards; 191 unit test files that touch the changed modules; `test_main` disk direct and ZMQ; official30 parity. ## New-case experiments diff --git a/tests/integration/test_chained_callable_lineage_journey.py b/tests/integration/test_chained_callable_lineage_journey.py index 96d3788bdd..50ad72c197 100644 --- a/tests/integration/test_chained_callable_lineage_journey.py +++ b/tests/integration/test_chained_callable_lineage_journey.py @@ -153,7 +153,7 @@ def test_chained_public_callable_uses_declared_main_flow_not_storage_argument( image_plan, labels_plan, rows_plan = invocation.artifact_output_plans records = [] for plan in (image_plan, labels_plan, rows_plan): - [record] = context.runtime_value_store.find_matching( + [record] = results.executed_contexts["A01"].runtime_value_store.find_matching( RuntimeArtifactQuery.from_output_plan( plan, axis_id="A01", backend="memory", group_key=plan.group_scope.select_runtime_key( diff --git a/tests/integration/test_custom_source_transport.py b/tests/integration/test_custom_source_transport.py index a4b8fab2df..33162103e5 100644 --- a/tests/integration/test_custom_source_transport.py +++ b/tests/integration/test_custom_source_transport.py @@ -23,7 +23,7 @@ SuccessProgressStatus, ) from tests.context_support import ( - executing_context, + compiled_context, ) WORKTREE = Path(__file__).resolve().parents[2] @@ -479,11 +479,10 @@ def _compiled_custom_contexts(payload, runtime): step_index=0, step_name="Synthetic custom", axis_id=axis_id, compiled_function_pattern=pattern, ) - context = executing_context( + context = compiled_context( axis_id=axis_id, step_plans={0: plan}, filemanager=FileManager({Backend.MEMORY.value: MemoryStorageBackend()}), ) - context.freeze() contexts[axis_id] = context normalized = FunctionStepTransportAuthority.normalize_context(context) transported = next( diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index 40529b06a0..259639e512 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -335,6 +335,7 @@ def test_photometry_carrier_preserves_raw_aliases_and_produced_pixels( if any(name in ("DNA", "Actin") for name in image_names): assert selected is None # Raw STEP_INPUT aliases need the current image carrier. results = _execute(tmp_path, document, bundle) + context = results.executed_contexts["A01"] assert results['A01'].is_success(), results['A01'].error_message (labels,) = context.runtime_value_store.find(name="Cells", axis_id="A01") foreground = object_label_dense_array(labels.data).squeeze() == 1 @@ -377,6 +378,7 @@ def test_label_only_measurement_keeps_stored_cohort(tmp_path): assert cohort is not None assert tuple(edge.spec.name for edge in cohort) == ("Cells",) results = _execute(tmp_path, document, bundle) + context = results.executed_contexts["A01"] assert results["A01"].is_success(), results["A01"].error_message (labels,) = context.runtime_value_store.find(name="Cells", axis_id="A01") (invocation,) = group.invocations @@ -437,6 +439,7 @@ def test_exact_secondary_selector_survives_authoring_compile_and_execution(tmp_p assert "labels" not in invocation.kwargs_dict results = _execute(tmp_path, reconstructed, bundle) + context = results.executed_contexts["A01"] assert results["A01"].is_success(), results["A01"].error_message store = context.runtime_value_store [primary] = store.find(name="Nuclei", axis_id="A01") @@ -507,7 +510,7 @@ def test_explicit_measurement_rosters_preserve_compiled_source_groups( ) document = PipelineDocumentCodec.from_source(PipelineDocumentCodec.render(document)) bundle = _compile(tmp_path, document, GlobalPipelineConfig(num_workers=1, use_threading=True)) - context = bundle.runtime_contexts["A01"] + context = bundle.runtime_contexts["A01"].executing() executor = FunctionStepExecutor(context, 2) prepared = executor._prepare_groups(executor._detect_patterns()) expected_keys = (None,) if producer_group_by is Ungrouped else ("1", "2") @@ -519,6 +522,7 @@ def test_explicit_measurement_rosters_preserve_compiled_source_groups( spec.name for spec in invocation.contract.artifact_inputs.of_artifact_type(ImageArtifactType) ) == ("DNA", "Actin") results = _execute(tmp_path, document, bundle) + context = results.executed_contexts["A01"] assert results["A01"].is_success(), results["A01"].error_message (output,) = invocation.artifact_output_plans measurements = context.runtime_value_store.find( @@ -617,7 +621,7 @@ def test_headless_entrypoint_requires_subject_and_executes_corrected_rows( bundle = _compile(tmp_path, document, global_config) results = _execute(tmp_path, document, bundle) assert results["A01"].is_success(), results["A01"].error_message - store = bundle.runtime_contexts["A01"].runtime_value_store + store = results.executed_contexts["A01"].runtime_value_store [counts] = store.find(name="PixelCounts", axis_id="A01") [image] = store.find(name="CountedImage", axis_id="A01") assert image.key.artifact_type is ImageArtifactType diff --git a/tests/integration/test_primary_segmentation_diagnostics_journey.py b/tests/integration/test_primary_segmentation_diagnostics_journey.py index 9ff0e70d64..35db917a80 100644 --- a/tests/integration/test_primary_segmentation_diagnostics_journey.py +++ b/tests/integration/test_primary_segmentation_diagnostics_journey.py @@ -304,7 +304,7 @@ def step(func, name, kwargs): ), ) assert results["A01"].is_success(), results["A01"].error_message - store = context.runtime_value_store + store = results.executed_contexts["A01"].runtime_value_store (primary_record,) = store.find(name="Nuclei", axis_id="A01") (secondary_record,) = store.find(name="Cells", axis_id="A01") source_metadata = primary_record.data.metadata From 807f1c9d8f3815a595b2615a4bb1ed781f10f04f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 16:24:47 -0400 Subject: [PATCH 09/13] WIP: K7 drop redundant plan scope type check Co-Authored-By: Claude Opus 5.5 --- openhcs/core/artifacts/plans.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openhcs/core/artifacts/plans.py b/openhcs/core/artifacts/plans.py index 5c2772b0de..ca301ab21d 100644 --- a/openhcs/core/artifacts/plans.py +++ b/openhcs/core/artifacts/plans.py @@ -313,11 +313,6 @@ def __post_init__(self) -> None: owner=f"Artifact plan {self.name!r}", ), ) - if not isinstance(self.group_scope, ComponentGroupScope): - raise TypeError( - f"Artifact plan {self.name!r} group_scope must be a " - f"ComponentGroupScope, got {type(self.group_scope).__name__}." - ) for grouped in self.group_scope.as_grouped(): if grouped.component in self.variable_components: raise ValueError( From e9daa2623eac89732be84277f82c6681251874c0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 16:31:03 -0400 Subject: [PATCH 10/13] WIP: K7 ratchet fixes Co-Authored-By: Claude Opus 5.5 --- openhcs/core/component_group_scope.py | 12 ++++++------ openhcs/core/orchestrator/bundle_execution.py | 3 ++- openhcs/core/orchestrator/execution_result.py | 15 +++++---------- openhcs/core/orchestrator/worker_execution.py | 2 +- openhcs/core/runtime_artifact_values.py | 8 ++++---- openhcs/core/steps/function_execution.py | 6 +++--- tests/unit/test_zmq_execution_server.py | 2 +- 7 files changed, 22 insertions(+), 26 deletions(-) diff --git a/openhcs/core/component_group_scope.py b/openhcs/core/component_group_scope.py index 17e909e6de..42c847d77d 100644 --- a/openhcs/core/component_group_scope.py +++ b/openhcs/core/component_group_scope.py @@ -80,10 +80,10 @@ def normalize_key(key: Hashable | None) -> ComponentGroupKey: @classmethod def one_per_component( cls, - scopes: Iterable["ComponentGroupScope"], + scopes: Iterable[ComponentGroupScope], *, owner: str, - ) -> tuple["GroupedScope", ...]: + ) -> tuple[GroupedScope, ...]: """Allow one consistent grouped scope per component.""" by_component: dict[type[Axis], GroupedScope] = {} for scope in scopes: @@ -99,7 +99,7 @@ def one_per_component( @staticmethod def union_compatible( - scopes: Sequence["ComponentGroupScope"], + scopes: Sequence[ComponentGroupScope], ) -> "ComponentGroupScope | None": """Union scopes of one component; None when their components differ.""" @@ -132,7 +132,7 @@ def require_single_key(self) -> str: ) @abstractmethod - def as_grouped(self) -> tuple["GroupedScope", ...]: + def as_grouped(self) -> tuple[GroupedScope, ...]: """Return this scope when it groups by a component, else nothing.""" def require_grouped(self, owner: str) -> "GroupedScope": @@ -237,7 +237,7 @@ class UngroupedScope(ComponentGroupScope): component: ClassVar[None] = None keys: ClassVar[tuple[None]] = (None,) - def as_grouped(self) -> tuple["GroupedScope", ...]: + def as_grouped(self) -> tuple[GroupedScope, ...]: return () def runtime_keys(self, discovered_keys): @@ -269,7 +269,7 @@ def __post_init__(self) -> None: if not is_axis(self.component): raise TypeError("ComponentGroupScope.component must be a declared axis.") - def as_grouped(self) -> tuple["GroupedScope", ...]: + def as_grouped(self) -> tuple[GroupedScope, ...]: return (self,) def require_grouped(self, owner: str) -> "GroupedScope": diff --git a/openhcs/core/orchestrator/bundle_execution.py b/openhcs/core/orchestrator/bundle_execution.py index 1d5394174c..ab87fe9653 100644 --- a/openhcs/core/orchestrator/bundle_execution.py +++ b/openhcs/core/orchestrator/bundle_execution.py @@ -43,6 +43,7 @@ from openhcs.core.orchestrator.execution_result import ( ExecutionResult, MergeAllRuntimeRecords, + OmitRuntimeRecords, RuntimeContextObservation, RuntimeExecutionObservation, RuntimeObservationMode, @@ -404,7 +405,7 @@ def validate_bundle_execution( dataset_step_indexes = validate_dataset_scoped_contexts(compiled_contexts) if ( dataset_step_indexes - and not request.runtime_observation_mode.collects_records + and request.runtime_observation_mode is OmitRuntimeRecords ): raise ValueError( "Dataset-scoped FunctionSteps require a runtime observation mode " diff --git a/openhcs/core/orchestrator/execution_result.py b/openhcs/core/orchestrator/execution_result.py index fb063a2712..80fe9d0ce0 100644 --- a/openhcs/core/orchestrator/execution_result.py +++ b/openhcs/core/orchestrator/execution_result.py @@ -5,6 +5,8 @@ following OpenHCS standards for explicit contracts and type safety. """ +from __future__ import annotations + import copyreg from abc import ABC, abstractmethod from dataclasses import dataclass, field, replace @@ -72,7 +74,7 @@ def from_context( context_key: str, context: ProcessingContext, records: tuple[StoredRuntimeValue, ...], - runtime_observation_mode: type["RuntimeObservationMode"], + runtime_observation_mode: type[RuntimeObservationMode], outputs: StepExecutionObservation, ) -> "RuntimeContextObservation": """Retain requested records beside the completed output projections.""" @@ -116,13 +118,10 @@ def merge_into(self, execution_contexts: Mapping[str, ExecutingContext]) -> None class RuntimeObservationMode(ABC): """Which worker runtime records an execution returns to the parent.""" - collects_records: ClassVar[bool] = True - returns_every_record: ClassVar[bool] = False - @classmethod def for_compiled_bundle( cls, bundle: "CompiledExecutionBundle" - ) -> type["RuntimeObservationMode"]: + ) -> type[RuntimeObservationMode]: """Consolidation carries rendered tables; only dataset-scoped inputs need records.""" if bundle.requires_parent_runtime_observation: @@ -133,7 +132,7 @@ def for_compiled_bundle( def including_parent_requirement( cls, required: bool, - ) -> type["RuntimeObservationMode"]: + ) -> type[RuntimeObservationMode]: """Return this mode strengthened by an additional retention requirement.""" return MergeAllRuntimeRecords if required else cls @@ -151,8 +150,6 @@ def retain_records( class OmitRuntimeRecords(RuntimeObservationMode): """Return no runtime records; the parent needs none.""" - collects_records = False - @classmethod def retain_records(cls, records, context): return () @@ -161,8 +158,6 @@ def retain_records(cls, records, context): class MergeAllRuntimeRecords(RuntimeObservationMode): """Return every runtime record to the parent.""" - returns_every_record = True - @classmethod def retain_records(cls, records, context): return records diff --git a/openhcs/core/orchestrator/worker_execution.py b/openhcs/core/orchestrator/worker_execution.py index 9c9d1dfeaa..0d6f3bf702 100644 --- a/openhcs/core/orchestrator/worker_execution.py +++ b/openhcs/core/orchestrator/worker_execution.py @@ -1470,7 +1470,7 @@ def _execute_single_axis_static( # original retention contract. Ordinary execution can retire dead inputs # only after the step's output/progress consumers have completed. release_consumed = ( - not runtime_observation_mode.returns_every_record + runtime_observation_mode is not MergeAllRuntimeRecords and isinstance(lane_context.debug_execution_policy, NoOpDebugExecutionPolicy) and not any( plan.visualize or plan.streaming_configs diff --git a/openhcs/core/runtime_artifact_values.py b/openhcs/core/runtime_artifact_values.py index 1a85cd56c2..5a6d4fb992 100644 --- a/openhcs/core/runtime_artifact_values.py +++ b/openhcs/core/runtime_artifact_values.py @@ -101,7 +101,7 @@ def from_spec( ) @classmethod - def compose(cls, values: Sequence["RuntimeValue"]) -> Any: + def compose(cls, values: Sequence[RuntimeValue]) -> Any: """Compose records through their shared artifact-type owner.""" runtime_values = cls._one_artifact_type(values) @@ -110,7 +110,7 @@ def compose(cls, values: Sequence["RuntimeValue"]) -> Any: @classmethod def compose_groups( cls, - values: Sequence["RuntimeValue"], + values: Sequence[RuntimeValue], producer_scope: GroupedScope, ) -> Any: """Rebuild the producer's group axis from one record per producer group.""" @@ -133,8 +133,8 @@ def compose_groups( @staticmethod def _one_artifact_type( - values: Sequence["RuntimeValue"], - ) -> tuple["RuntimeValue", ...]: + values: Sequence[RuntimeValue], + ) -> tuple[RuntimeValue, ...]: runtime_values = tuple(values) if not runtime_values: raise ValueError("Cannot compose an empty runtime value group.") diff --git a/openhcs/core/steps/function_execution.py b/openhcs/core/steps/function_execution.py index a0b9017836..88901860ff 100644 --- a/openhcs/core/steps/function_execution.py +++ b/openhcs/core/steps/function_execution.py @@ -593,7 +593,9 @@ def candidate_scopes( variable_components=ComponentSet.of(plan.variable_components), ) - if not scope.has_compiled_keys: + if scope.has_compiled_keys: + component_keys = scope.runtime_keys(()) + else: discovered_keys = [] for group in pattern.groups: for invocation in group.invocations: @@ -611,8 +613,6 @@ def candidate_scopes( if not any(key is not None for key in discovered_keys): return None component_keys = scope.runtime_keys(discovered_keys) - else: - component_keys = scope.runtime_keys(()) selected_groups = [] for component_key in component_keys: diff --git a/tests/unit/test_zmq_execution_server.py b/tests/unit/test_zmq_execution_server.py index 0ee0daf60b..b56e033044 100644 --- a/tests/unit/test_zmq_execution_server.py +++ b/tests/unit/test_zmq_execution_server.py @@ -263,7 +263,7 @@ def test_zmq_auxiliary_params_strengthen_compiled_observation_requirement( else None ) execution_bundle = SimpleNamespace( - requires_parent_runtime_observation=compiled_mode.collects_records, + requires_parent_runtime_observation=compiled_mode is not OmitRuntimeRecords, ) assert params.runtime_observation_mode_for(execution_bundle) is expected_mode From f528cb67f18fccd841da7999a9f8919cd18ebdb0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 16:36:32 -0400 Subject: [PATCH 11/13] K7: receipt outcome Co-Authored-By: Claude Opus 5.5 --- .../consolidation-20261009/K7-execution-protocol.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/refactor/consolidation-20261009/K7-execution-protocol.md b/docs/refactor/consolidation-20261009/K7-execution-protocol.md index df0cbfb3f9..8a56c5e320 100644 --- a/docs/refactor/consolidation-20261009/K7-execution-protocol.md +++ b/docs/refactor/consolidation-20261009/K7-execution-protocol.md @@ -27,7 +27,7 @@ Re-measured at this head. G7 replaced part of K7 (see Already done); the other t - **Phase types** (`core/context/processing_context.py`). `ProcessingContext` is the base (`require_axis_id`, `require_parameter_name`). `CompilingContext` is the only mutable phase; `freeze()` returns a `CompiledContext`, which rejects every attribute assignment and is what bundles, transport and workers carry. `CompiledContext.executing(lane)` starts one execution: `ExecutingContext` adds the runtime value store, the four runtime caches, the debug sink, the lane runtime, completed outputs and the per-step value scope, and still rejects assignment to compiled facts. A worker lane builds one per compiled context it executes; the parent builds one per compiled context per `execute_compiled_bundle` call, merges worker observations into it, runs dataset-scoped steps and post-execute hooks on it, and returns them as `BundleExecutionResults.executed_contexts` (the ZMQ observation export reads them there, not from the bundle). Deleted: `_is_frozen`, `is_frozen()`, the six compiler checks and the worker check, `bind_execution_runtime`, `reset_completed_step_outputs`, `inject_plan`, `visualizer_config` (always `None`) and the `core.context` re-export. - **One result family** (`core/orchestrator/execution_result.py`). `ExecutionResult` ABC with `ExecutionSucceeded`, and `ExecutionStopped` (owns `error_message`, `failed_combination`, `progress_phase`, `progress_status`, `at_combination`) with `ExecutionFailed` and `ExecutionCancelled` (`stops_lane`). Deleted: `ExecutionStatus`, `is_error`, `is_cancelled`, the three constructors, and `ZMQExecutionAxisOutcome` (the outcome export stores `result.without_runtime_observation()`; schema version 5). `RuntimeObservationMode` is a family: `OmitRuntimeRecords`, `MergeAllRuntimeRecords`, `MergeDatasetInputRecords` (E1's names), each owning `retain_records`. -Production (Python under `openhcs/`): see the PR status line. The net is small because three families replace three encodings one for one; the deletions are the predicates, re-packs and checks, the additions are the variant classes. +Production (Python under `openhcs/`): −1331 +1248. The net is small because three families replace three encodings one for one; the deletions are the predicates, re-packs and checks, the additions are the variant classes. ## Deletion certificates @@ -82,10 +82,8 @@ Tests take the new constructors (`tests/context_support.py` builds phase context The guards pass; touched tests pass; official30 is 30/30; the structural ratchet and nra-r1 show zero regressions. -## Progress +## Outcome -- [x] Receipt. -- [x] ExecutionResult and RuntimeObservationMode families (`ExecutionSucceeded`/`ExecutionFailed`/`ExecutionCancelled`; `OmitRuntimeRecords`/`MergeAllRuntimeRecords`/`MergeDatasetInputRecords`; `ZMQExecutionAxisOutcome` deleted, outcome export stores results without records). -- [ ] ComponentGroupScope variants, plan `group_scope` field, compose split. -- [ ] Context phase types. -- [ ] Guards, tests, ratchet, parity, PR. +- Scope predicate reads outside the family: 48 before, 0 after (AST scan). `is None` checks on a scope's component in the compiler: 2 before, 0 after. +- Production (Python under `openhcs/`): −1331 +1248. Tests: −1462 +1684 (the guard file, the context helper and wider import blocks). +- Structural ratchet (`openhcs`, `scripts`, `benchmark`) and nra-r1 against `d597aa224`: no increase. From a9ef16e09ac730881a4fff7c8545bb3814b4af15 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 17:09:40 -0400 Subject: [PATCH 12/13] K7: receipt after merging main (E1 resolution, input_plan, axis-scope follow-up, caches) Co-Authored-By: Claude Opus 5.5 --- .../consolidation-20261009/K7-execution-protocol.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/refactor/consolidation-20261009/K7-execution-protocol.md b/docs/refactor/consolidation-20261009/K7-execution-protocol.md index 8a56c5e320..0c3a2603fa 100644 --- a/docs/refactor/consolidation-20261009/K7-execution-protocol.md +++ b/docs/refactor/consolidation-20261009/K7-execution-protocol.md @@ -44,14 +44,14 @@ Production (Python under `openhcs/`): −1331 +1248. The net is small because th | `context.visualizer_config` and the `vis_config` argument at the one call | **derived** (it was always `None`, the parameter's default) | | `ExecutionStatus`, `is_error`, `is_cancelled`, `ExecutionResult.success/error/cancelled` | **derived** (the result subclasses) | | `ZMQExecutionAxisOutcome` | **derived** (`ExecutionResult.without_runtime_observation`) | -| `RuntimeObservationMode.from_parent_requirement` | **unused** | +| `NamedSourceBinding.input_plan` (`core/source_bindings.py`) | **unused** (no caller in `openhcs/`, tests, benchmark or scripts) | | `openhcs.core.context.ProcessingContext` re-export | **contract** (no re-exports) | **Tests-only** (behaviour the interface admits that no test exercises): - `ComponentGroupScope.of` removes duplicate keys and reads an empty key tuple as "no named keys"; `from_raw` kept duplicates and rejected an empty tuple. No producer of duplicate or empty keys is tested. - `RuntimeArtifactInput.records` no longer special-cases an ungrouped producer; the general path selects the same record. A relation that declares a complete-producer selection on an ungrouped producer now runs the declaration check it previously skipped. - `ArtifactInputProjectionPlan.validate_storage_plan` raises one message for every out-of-scope selection (three before). -- Runtime caches (`runtime_image_stack_cache`, pattern-discovery, source-binding, output-identity) now live for one execution of a compiled context. Before, in-process lanes and repeated executions of one bundle shared them through the compiled context. Reuse across executions of one bundle is untested; correctness does not depend on it, timing might. +- Runtime caches (`runtime_image_stack_cache`, pattern-discovery, source-binding, output-identity) now live for one execution of a compiled context. They were already per compiled context (one per axis value, or per axis value and sequential combination), so no cache that axes shared is now rebuilt per axis; every reader is axis-scoped step execution (`function_execution.py`, `function_runtime.py`, `source_binding_selection.py`). What is lost: when one in-memory bundle is executed again with inline or thread lanes, the pattern-discovery, source-binding and output-identity caches used to start warm (the image cache and the value store were cleared each time already). Process lanes always started cold, from the pickled bundle. Re-execution of one bundle with warm caches is untested. - Thread and inline lanes no longer execute on the parent's context object; they get their own `ExecutingContext` like process lanes, and only returned observations reach the parent. Code that read a parent context's store mid-execution from another thread is untested (none found). - A cancelled or failed axis reports `type(result).__name__` in `require_success` and export messages (`ExecutionFailed: …`), not `error: …`. @@ -59,10 +59,9 @@ Production (Python under `openhcs/`): −1331 +1248. The net is small because th | Item | Evidence | Belongs to | |---|---|---| -| `NamedSourceBinding.input_plan(group_keys=, group_component=)` has no caller and names the deleted plan fields | `core/source_bindings.py:1298` | E1 (owns the file in #1203): delete | | Annotations `ProcessingContext` on functions that read `execution_runtime` | `processing/materialization/core.py`, `path_scopes.py` | K8: annotate `ExecutingContext` | -| `RuntimeExecutionAxisScope(component=None, value=None)` packs "no group coordinate" the same way the scope did | `core/component_group_scope.py` | not assigned; for the session lead | -| E1 (#1203) also converts `ExecutionStatus` and `RuntimeObservationMode`; this surface uses E1's mode names and replaces the status family with result subclasses | `execution_result.py` | whichever merges second resolves in favour of the result family | +| `RuntimeExecutionAxisScope(component=None, value=None)` packs "no group coordinate" the same way the scope did. Not small: 88 production references in 16 files and 388 test references, and it is the `scope` of every runtime record key (`key.scope.component` is compared across `runtime_stores.py`, `kinds.py`, materialization) | `core/component_group_scope.py` | follow-up surface, not started here | +| E1 (#1203, merged first) converted `ExecutionStatus` and `RuntimeObservationMode` | `execution_result.py` | resolved in the merge of `625150118`: E1's observation-mode family is kept as written; its `ExecutionStatus`/`AxisSucceeded`/`AxisFailed`/`AxisCancelled` family is deleted for the result subclasses and its readers migrated. E1's guard table had no row for anything deleted here | | `PipelineOrchestrator.execute_compiled_bundle(runtime_observation_mode=None)` derives the mode when `None` | `orchestrator.py:765` | G7 follow-up, not assigned | ## Guards From 8641a6eb343f2f5502ec22001e169dd07d102f4c Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 11 Oct 2026 21:07:34 -0400 Subject: [PATCH 13/13] K7: publication journey tests execute through ExecutingContext; restore progress fixture import Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_artifact_publication_journey.py | 7 +++++-- tests/unit/test_named_checkpoint_publication.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_artifact_publication_journey.py b/tests/unit/test_artifact_publication_journey.py index 8a362ad4e7..d7e0785f37 100644 --- a/tests/unit/test_artifact_publication_journey.py +++ b/tests/unit/test_artifact_publication_journey.py @@ -168,13 +168,16 @@ def test_converted_checkpoint_has_typed_address_on_final_reconciliation(tmp_path results = tmp_path / "source_out" / "results" if absolute else Path("results") orchestrator, steps, bundle = _compile(tmp_path, results, main_filter) # Workers return each step's saved outputs; the receiving boundary publishes them. + contexts = { + key: context.executing() for key, context in bundle.runtime_contexts.items() + } observations = RuntimeExecutionObservation(contexts=tuple( RuntimeContextObservation(context_key=key, records=(), outputs=step.process(context, index)) - for key, context in bundle.runtime_contexts.items() + for key, context in contexts.items() for index, step in enumerate(steps) )) OpenHCSMetadataTarget.finalize_completed_dataset( - bundle.runtime_contexts, runtime_observations=(observations,) + contexts, runtime_observations=(observations,) ) saved = tmp_path / "source_out" / "converted" files = tuple(saved.glob("*.tif")) diff --git a/tests/unit/test_named_checkpoint_publication.py b/tests/unit/test_named_checkpoint_publication.py index 9a6b1f982a..1fff1b62bd 100644 --- a/tests/unit/test_named_checkpoint_publication.py +++ b/tests/unit/test_named_checkpoint_publication.py @@ -12,7 +12,7 @@ import pytest import tifffile -from test_artifact_publication_journey import _plate +from test_artifact_publication_journey import _plate, progress_events # noqa: F401 (fixture) from openhcs.core.artifacts.sidecars import QaCheckpoint from openhcs.core.artifacts.kinds import ImageArtifactType from openhcs.core.artifacts.specs import (