From 3ed81cd633a8b6cffe99ffe6a9cc57a1e766cff0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 14:23:59 -0400 Subject: [PATCH 01/17] Validate measurement subjects at compile and expose declared selectors --- ...easurement_contract_checkpoint_20260929.md | 52 +++++++++++ .../services/function_catalog_service.py | 4 +- openhcs/core/artifact_key_selection.py | 5 +- openhcs/core/artifacts.py | 65 +++++++++++--- openhcs/core/function_patterns.py | 3 + openhcs/core/steps/function_runtime.py | 8 +- ...cellprofiler_function_contract_friction.py | 33 +++++++ tests/unit/test_function_artifact_outputs.py | 87 +++++++++++++++++++ 8 files changed, 233 insertions(+), 24 deletions(-) create mode 100644 docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md new file mode 100644 index 0000000000..7a7d0df7c1 --- /dev/null +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -0,0 +1,52 @@ +# Measurement declaration checkpoint (#133, #204) + +Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. +Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. +Base: `openhcsdev/main` at `98d9b9d23`, fetched and merged normally on 2026-09-29. + +## Implemented + +- Measurement output validation belongs to `MeasurementsArtifactType`. Compilation + requires a subject relation before invoking the callable. Schema-bearing rows and + CSV materialization do not imply a measurement subject. +- Compile and runtime resolve subjects through the original relation owners. The + error identifies the output and explains image, object, and artifact relations. +- CellProfiler catalog bindings project `require_parameter_name()`, not the optional + raw `parameter_name`. The existing `select_object_sets_to_measure` selector is + discoverable separately from the runtime-owned scalar `labels` parameter. +- Catalog guidance explains exact-name tuples and consumed compile-time selectors. + No selector registry, artifact store, fallback, or ambiguity relaxation was added. + +## Focused evidence + +Using `/home/ts/code/projects/openhcs/.venv/bin/python` with this worktree and all +eight recorded submodule `src` directories on `PYTHONPATH`; all nine imported +package paths were verified inside this worktree. Submodules were initialized to +their recorded gitlinks without installed-source changes. + +- `pytest -q tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py + tests/unit/test_cellprofiler_artifact_identity_reconstruction.py`: 12 passed. +- `pytest -q tests/unit/test_function_artifact_outputs.py -k + 'measurement_subject or wraps_columnar_rows_with_compiled_measurement_identity'`: + 6 passed, 61 deselected. Missing and conflicting subjects fail before execution; + the corrected image-subject declaration compiles and executes columnar rows. +- Runs used nonblocking batch validation `flock` and CPU-only mode. The resource + guard reported 12.5 GiB available RAM and a warning for 11.7 GiB used swap. + No large suite, JVM, MCP, or extra GUI was started. + +Architecture review was focused source/ownership inspection, not a complete NRA +scan. It follows the NRA and current refactor-audit declaration-ownership rules. + +## Remaining acceptance + +- Continuous synthetic two-producer compile, exact-selector document roundtrip, + runtime binding, and omitted-selector ambiguity regression for #204. +- Realistic current headless compile-inspection entrypoint for both issues. +- Coordinator-authorized actual UI code-document/live installation validation. + Source tests do not establish installed readiness. Worker must not install or + merge, and must not inspect Euler's blind input or output. +- Correct #204's diagnosis after the complete selector proof; retain its original + observed failure. Both issues remain referenced, not claimed closed here. + +No owned large scratch artifacts have been created. Viewer-bind and ROI +materialization implementation surfaces remain with their assigned owners. diff --git a/openhcs/agent/services/function_catalog_service.py b/openhcs/agent/services/function_catalog_service.py index f167c93318..79f968779e 100644 --- a/openhcs/agent/services/function_catalog_service.py +++ b/openhcs/agent/services/function_catalog_service.py @@ -1081,7 +1081,7 @@ def _cellprofiler_artifact_binding_summary( direction="input" if plan_type is ArtifactInputPlan else "output", kind=binding.require_artifact_type().require_value(), setting_names=setting_names(binding.setting_name), - parameter_name=binding.parameter_name, + parameter_name=binding.require_parameter_name(), runtime_parameter_name=binding.runtime_parameter_name, repeated=binding.repeated, ) @@ -1098,7 +1098,7 @@ def _source_binding_rule( contract: CallableContract, ) -> str | None: if cellprofiler_module is not None: - return "CellProfiler exact artifact names are resolved from the module declaration, concrete FunctionStep groups, and compile-time setting identities. Callable-level artifact arrays can therefore be empty before compilation; inspect the module artifact_bindings here and the compiled artifact plan for exact names." + return "CellProfiler exact artifact names are resolved from the module declaration, concrete FunctionStep groups, and compile-time setting identities. Callable-level artifact arrays can therefore be empty before compilation; inspect the module artifact_bindings here and the compiled artifact plan for exact names. Author selectors in FunctionStep func kwargs using each binding's parameter_name; for repeated bindings, use a tuple of exact artifact names (a one-element tuple selects one producer). These compile-time selectors are consumed by the declaration, not passed to the callable. runtime_parameter_name is runtime-owned: do not pass labels or other runtime payloads as function kwargs. When multiple label producers are available, select the exact intended output name; omission remains ambiguous and fails closed." if contract.artifact_inputs: return "Artifact input bindings are resolved from canonical CallableContract artifact_inputs during compilation." return None diff --git a/openhcs/core/artifact_key_selection.py b/openhcs/core/artifact_key_selection.py index 2de3371264..fb0d9f0473 100644 --- a/openhcs/core/artifact_key_selection.py +++ b/openhcs/core/artifact_key_selection.py @@ -56,7 +56,10 @@ def select_plans( return tuple(selected) def validate_artifact_relation_refs(self, *, owner_name: str) -> None: + output_specs = self.artifact_specs.for_plan_type(ArtifactOutputPlan).specs + for spec in output_specs: + spec.artifact_type.validate_output_declaration(spec) self.artifact_specs.validate_registered_relation_refs( owner_name=owner_name, - relation_specs=self.artifact_specs.for_plan_type(ArtifactOutputPlan).specs, + relation_specs=output_specs, ) diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index 72fe223dd8..203f28ebfa 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -142,6 +142,12 @@ def diagnostic_label(cls) -> str: """Return the stable artifact type label used in diagnostics.""" return f"<{cls.__name__}: {cls.require_value()!r}>" + @classmethod + def validate_output_declaration(cls, spec: "ArtifactSpec") -> None: + """Validate kind-owned output requirements before runtime planning.""" + + del spec + @classmethod def normalize_group_scoped_payload( cls, @@ -664,6 +670,29 @@ class MeasurementsArtifactType(ArtifactType): value = "measurements" payload_shape = ArtifactPayloadShape.TABLE + @classmethod + def require_output_subject( + cls, + output: "ArtifactSpec | ArtifactOutputPlan", + ) -> "MeasurementSubject": + """Require the same declared subject at compile and runtime boundaries.""" + + subject = ArtifactSpecRelation.measurement_subject_for_output(output) + if subject is None: + raise ValueError( + f"Measurement output {output.ref()!r} has no declared " + "measurement subject relation. Add ImageMeasurementSubjectRelation " + "with the exact declared image ref for image-level rows, " + "ObjectMeasurementSubjectRelation with the exact object-label ref " + "for object-level rows, or ArtifactMeasurementSubjectRelation() " + "for artifact-level rows to ArtifactSpec.output(..., relations=(...,))." + ) + return subject + + @classmethod + def validate_output_declaration(cls, spec: "ArtifactSpec") -> None: + cls.require_output_subject(spec) + @classmethod def runtime_parameter_types(cls) -> tuple[type, ...]: from openhcs.core.runtime_tabular_values import ColumnarRows @@ -1210,6 +1239,27 @@ def measurement_subject(self) -> "MeasurementSubject | None": return None + @staticmethod + def measurement_subject_for_output( + output: "ArtifactSpec | ArtifactOutputPlan", + ) -> "MeasurementSubject | None": + """Resolve the sole subject from an output's original relation owners.""" + + subjects = tuple( + dict.fromkeys( + subject + for relation in output.relations + for subject in (relation.measurement_subject(),) + if subject is not None + ) + ) + if len(subjects) > 1: + raise ValueError( + f"Artifact output {output.ref()!r} declares multiple measurement " + f"subjects: {subjects!r}." + ) + return subjects[0] if subjects else None + def object_subject_binding(self) -> ObjectArtifactSubjectBinding | None: """Return the target-local identity bound to one object subject.""" @@ -2731,20 +2781,7 @@ def source_context_source(self) -> ArtifactSpecRef | None: def measurement_subject(self) -> "MeasurementSubject | None": """Return the sole measurement subject declared by this output.""" - subjects = tuple( - dict.fromkeys( - subject - for relation in self.relations - for subject in (relation.measurement_subject(),) - if subject is not None - ) - ) - if len(subjects) > 1: - raise ValueError( - f"Artifact output {self.ref()!r} declares multiple measurement " - f"subjects: {subjects!r}." - ) - return subjects[0] if subjects else None + return ArtifactSpecRelation.measurement_subject_for_output(self) def object_subject_binding(self) -> ObjectArtifactSubjectBinding | None: """Return the sole object-subject binding declared by this output.""" diff --git a/openhcs/core/function_patterns.py b/openhcs/core/function_patterns.py index b828123d2f..bb45eadce1 100644 --- a/openhcs/core/function_patterns.py +++ b/openhcs/core/function_patterns.py @@ -1223,6 +1223,9 @@ def _compile_invocation( ) item = replace(item, contract=contract_plan.contract) artifact_selector = declaration_provider(item, step_context) + artifact_selector.validate_artifact_relation_refs( + owner_name=item.contract.function_name, + ) artifact_input_plans = artifact_selector.select_plans( ArtifactInputPlan, input_plans, diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index 440c03ca4b..4b33f735c5 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -629,13 +629,7 @@ class MeasurementsFunctionOutputContextStrategy(ProjectedFunctionOutputContextSt def _declared_subject(output_plan: ArtifactOutputPlan | None) -> MeasurementSubject: if output_plan is None: raise ValueError("Measurement outputs require a compiled output plan.") - subject = output_plan.measurement_subject() - if subject is None: - raise ValueError( - f"Measurement output {output_plan.ref()!r} has no declared " - "measurement subject relation." - ) - return subject + return MeasurementsArtifactType.require_output_subject(output_plan) @staticmethod def _validate_nominal_table( diff --git a/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py b/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py index b5cbbbd00c..729d931959 100644 --- a/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py +++ b/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py @@ -12,6 +12,10 @@ identify_primary_objects, ) from openhcs.processing.backends.cellprofiler.skeleton import measure_object_skeleton +from openhcs.processing.backends.cellprofiler.intensity import ( + MeasureObjectIntensityModule, + measure_object_intensity, +) @dataclass(frozen=True) @@ -156,3 +160,32 @@ def test_function_detail_classifies_normalized_artifact_fed_parameter(monkeypatc assert "do not pass this as a function kwarg" in ( parameters["seed_labels"].description or "" ) + + +def test_catalog_exposes_declared_exact_object_selector_and_authoring_guidance( + monkeypatch, +): + catalog = _catalog( + monkeypatch, + "openhcs:cellprofiler_measure_object_intensity", + measure_object_intensity, + ) + detail = catalog.get("openhcs:cellprofiler_measure_object_intensity") + contract = detail.runtime_contract + assert contract is not None + assert contract.cellprofiler_module is not None + (selector,) = ( + binding + for binding in contract.cellprofiler_module.artifact_bindings + if binding.runtime_parameter_name == "labels" + ) + owner = MeasureObjectIntensityModule.object_measurement_binding + assert owner.parameter_name is None + assert selector.parameter_name == owner.require_parameter_name() + assert selector.parameter_name == "select_object_sets_to_measure" + assert selector.repeated is True + assert contract.source_binding_rule is not None + assert "one-element tuple selects one producer" in contract.source_binding_rule + assert "do not pass labels" in contract.source_binding_rule + parameters = {parameter.name: parameter for parameter in detail.parameters} + assert parameters["labels"].supplied_by is FunctionParameterSource.ARTIFACT_INPUT diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index f03639686f..55d53509e0 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -16,6 +16,7 @@ ArtifactSpecRef, GroupLineageSourceRelation, ImageArtifactType, + ImageMeasurementSubjectRelation, ObjectLabelsArtifactType, MeasurementsArtifactType, MetadataArtifactType, @@ -51,6 +52,7 @@ composed_image_payload, special_outputs, ) +from openhcs.core.pipeline.artifact_planning import extract_artifact_declarations from openhcs.core.function_contract_metadata import FunctionContractAttribute from openhcs.core.pipeline.path_planner import PathPlanner, PathPlannerArtifactStage from openhcs.core.source_bindings import ( @@ -114,6 +116,7 @@ ) from openhcs.processing.backends.assemblers.assemble_stack_cpu import assemble_stack_cpu from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.processing.materialization import CsvOptions, MaterializationSpec from openhcs.processing.backends.analysis.multi_template_matching import ( TemplateMatchResult, ) @@ -2317,6 +2320,90 @@ class _NativeCountRow: cell_count: int +@pytest.mark.parametrize("boundary", ["inspection", "compile"]) +def test_measurement_subject_is_required_before_schema_rows_execute(boundary): + spec = ArtifactSpec.output( + "cell_counts", + MeasurementsArtifactType, + materialization=MaterializationSpec(CsvOptions()), + ) + + @artifact_outputs(spec) + def count(image): + raise AssertionError("Invalid measurement declarations must not execute") + + with pytest.raises(ValueError, match="cell_counts.*no declared measurement subject") as exc: + if boundary == "inspection": + extract_artifact_declarations(count) + else: + compile_function_pattern(count, {}, {}) + assert "ImageMeasurementSubjectRelation" in str(exc.value) + assert "ArtifactSpec.output" in str(exc.value) + + +def test_corrected_image_measurement_subject_compiles_and_executes_columnar_rows(): + image = ArtifactSpec.output("CountedImage", ImageArtifactType) + rows = ArtifactSpec.output( + "cell_counts", + MeasurementsArtifactType, + materialization=MaterializationSpec(CsvOptions()), + relations=(ImageMeasurementSubjectRelation(image.ref()),), + ) + + @artifact_outputs(image, rows) + def count(image): + return image, DataclassMeasurementColumnarRows((_NativeCountRow(0, 2),)) + + graph = extract_artifact_declarations(count) + assert graph.outputs[rows.ref()] == rows + context = ContextStub() + _execute_function_core( + CoreExecutionRequest( + func_callable=count, + main_data_arg=ImagePayloadMetadata( + source_path="/input/A01_s1_w1.tif" + ).payload_with(np.zeros((1, 4, 5), dtype=np.uint16)), + base_kwargs={}, + context=context, + artifact_inputs={}, + artifact_outputs={ + spec.ref(): ArtifactOutputPlan( + spec.name, + f"/memory/{spec.name}.pkl", + artifact_type=spec.artifact_type, + relations=spec.relations, + ) + for spec in (image, rows) + }, + ) + ) + [stored] = context.runtime_value_store.find(name=rows.name, axis_id=context.axis_id) + assert isinstance(stored.value.data, MeasurementTable) + assert stored.value.data.subject == MeasurementSubject( + MeasurementScope.IMAGE, image.name + ) + assert tuple(stored.value.data.rows.column_values("cell_count")) == (2,) + + +def test_compile_rejects_conflicting_measurement_subject_relations(): + image = ArtifactSpec.output("CountedImage", ImageArtifactType) + rows = ArtifactSpec.output( + "cell_counts", + MeasurementsArtifactType, + relations=( + ImageMeasurementSubjectRelation(image.ref()), + ArtifactMeasurementSubjectRelation(), + ), + ) + + @artifact_outputs(image, rows) + def count(image): + return image + + with pytest.raises(ValueError, match="multiple measurement subjects"): + compile_function_pattern(count, {}, {}) + + def test_execute_function_core_wraps_columnar_rows_with_compiled_measurement_identity(): context = ContextStub() measurement_spec = ArtifactSpec.output( From e56b977a54fd48ca7989ac27d1740fd3a9e54174 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 14:47:25 -0400 Subject: [PATCH 02/17] Verify measurement declaration journeys and adapter output ownership --- ...easurement_contract_checkpoint_20260929.md | 118 +++++- .../issue204_catalog_diagnosis_20260929.md | 88 +++++ openhcs/agent/dto/functions.py | 2 +- openhcs/core/artifact_key_selection.py | 11 +- openhcs/core/callable_contract.py | 8 + openhcs/core/function_patterns.py | 4 +- .../test_measurement_declaration_journey.py | 344 ++++++++++++++++++ ...cellprofiler_function_contract_friction.py | 35 ++ tests/unit/test_function_artifact_outputs.py | 14 +- 9 files changed, 597 insertions(+), 27 deletions(-) create mode 100644 docs/plans/issue204_catalog_diagnosis_20260929.md create mode 100644 tests/integration/test_measurement_declaration_journey.py diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 7a7d0df7c1..7d5847113a 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -3,6 +3,11 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. Base: `openhcsdev/main` at `98d9b9d23`, fetched and merged normally on 2026-09-29. +Fetched again before publishing the regression checkpoint; normal merge reported +already up to date. No parent/integration edits, rebases, resets, or gitlink changes. + +Closes #133. References #204; its paired-channel and actual GUI acceptance remain +open. This is a source-verified draft, not an installed/live-readiness claim. ## Implemented @@ -11,6 +16,10 @@ Base: `openhcsdev/main` at `98d9b9d23`, fetched and merged normally on 2026-09-2 CSV materialization do not imply a measurement subject. - Compile and runtime resolve subjects through the original relation owners. The error identifies the output and explains image, object, and artifact relations. +- The finalized callable contract delegates adapter-recorded outputs to the + existing runtime adapter owner (`manages_artifact_outputs`). Native returns use + the artifact-kind requirement. This distinction is required: CellProfiler + primary/secondary modules record heterogeneous measurements themselves. - CellProfiler catalog bindings project `require_parameter_name()`, not the optional raw `parameter_name`. The existing `select_object_sets_to_measure` selector is discoverable separately from the runtime-owned scalar `labels` parameter. @@ -24,29 +33,108 @@ eight recorded submodule `src` directories on `PYTHONPATH`; all nine imported package paths were verified inside this worktree. Submodules were initialized to their recorded gitlinks without installed-source changes. -- `pytest -q tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py - tests/unit/test_cellprofiler_artifact_identity_reconstruction.py`: 12 passed. -- `pytest -q tests/unit/test_function_artifact_outputs.py -k - 'measurement_subject or wraps_columnar_rows_with_compiled_measurement_identity'`: - 6 passed, 61 deselected. Missing and conflicting subjects fail before execution; - the corrected image-subject declaration compiles and executes columnar rows. +- Initial published checkpoint: 12 catalog/identity tests and 6 focused native + subject tests passed. +- Full focused rerun: `pytest -q tests/unit/test_function_artifact_outputs.py + tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py + tests/unit/test_cellprofiler_artifact_identity_reconstruction.py + tests/integration/test_measurement_declaration_journey.py -k 'not paired'`: + **85 passed, 1 deselected**, 9.33 seconds (80 unit, 5 real headless journey cases). +- The real `PipelineDocumentAuthority` and `FunctionStepDocumentAuthority` + render/parse routes preserve `select_object_sets_to_measure=("Cells",)`. + `InProcessCompileInspectionGateway.compile` selects exactly the secondary + producer (source step 1) for runtime `labels`; authored selectors are consumed. +- The same-source two-producer journey then runs the real orchestrator and + verifies the secondary label area is larger than the primary area, the returned + measurement subject is `Cells`, and integrated DNA intensity includes the grown + secondary region rather than just the four bright primary pixels. Feature + retrieval uses the existing CellProfiler measurement dialect. +- The paired DNA/actin-channel case also passes roundtrip and compile, but its + earlier continuous run failed at the secondary step's primary-label input. + It remains an unweakened failing regression, deselected explicitly above; + see the remaining acceptance section. No xfail or fallback was added. +- Omitted selection with two producers fails closed with the original useful + ambiguity diagnostic; omitted selection with one producer compiles. +- Missing and conflicting native subjects fail before execution. The real + headless compiler rejects schema-bearing `DataclassMeasurementColumnarRows` + with CSV options but no subject. The corrected image-subject declaration + compiles and executes, returning an image-owned table with four counted pixels. - Runs used nonblocking batch validation `flock` and CPU-only mode. The resource - guard reported 12.5 GiB available RAM and a warning for 11.7 GiB used swap. - No large suite, JVM, MCP, or extra GUI was started. + guard most recently reported 14.7 GiB available RAM and a warning for historical + 11.5 GiB used swap. No broad suite, JVM, MCP server, or extra GUI was started. + The final run completed and released the lock before Euler's H003 restart; + no further heavy validation was started after the resource-priority instruction. + +## Essential anti-pattern audit: actual ownership evidence + +This is a focused source/caller and AST audit against the current archived +refactor-audit pattern catalog, **not a complete NRA semantic/proof scan**. All +seven changed production files were parsed against the merged base: zero added +`match`, `isinstance`, `getattr`/`hasattr`, or 6+-term boolean-chain AST nodes. +That guard detects listed shapes, not every possible semantic ownership defect. + +| Pattern IDs | Owner, projection, and new-case evidence | +| --- | --- | +| IMPL-1/2/3/4, MEMB-1/2 | The finalized compiler calls `ArtifactPlanKeySelector.validate_artifact_output_declarations`; each spec delegates to `spec.artifact_type.validate_output_declaration`. Only `MeasurementsArtifactType` owns the subject requirement. No kind/name roster or external type switch was added. A future output kind supplies its rule on its declaration; compile/runtime consumers need no new branch. | +| IMPL-5/12/13, TIME-1/3 | `ArtifactSpecRelation.measurement_subject_for_output` resolves original relation-owned subjects once. The output-plan accessor, kind requirement, and native runtime all delegate; the former output-plan extraction body and runtime missing-subject body were deleted in place. The output-plan method remains a domain accessor, not an old reader or compatibility alias. | +| IDEN-1/2/5, BOUND-2/7, MEMB-3/4 | `SettingToKeywordBinding.require_parameter_name()` owns selector identity, separately from `runtime_parameter_name`. The only production `CellProfilerArtifactBindingSummary` constructor projects that owner; its selector field is now required. No selector store, guessed names, raw-row decoder, or attribute-name probe exists in the change. The catalog regression derives every registered binding and exercises new derived-name input and explicit-name output declarations without editing the projection. | +| IDEN-7, IMPL-10 | An initial placement incorrectly rejected heterogeneous CellProfiler records. Validation now runs after invocation-contract finalization, where the existing `runtime_adapter.manages_artifact_outputs` capability names the output owner. Native table-wide subject invariants apply only to native returned payloads, not adapter-recorded tables. Real primary/secondary compilation and execution plus native missing/corrected tests exercise both owners. | +| AGENT-2/3/6/7, TIME-5/9 | No second mechanism, format converter, test-only production switch, or dormant feature was added. One pre-existing cardinality fixture received a valid artifact-level subject so its original missing-return assertion remains meaningful. The paired-channel failure is retained, not hidden by weaker expectations; publication proceeds with that exact acceptance gap stated. | -Architecture review was focused source/ownership inspection, not a complete NRA -scan. It follows the NRA and current refactor-audit declaration-ownership rules. +Caller review covered the default invocation declaration provider (returns the +finalized `CallableContract`), every call to the new output-validation hook and +shared subject resolver, all production measurement-subject accessors, and the +catalog DTO constructor/consumers. Structural `extract_artifact_declarations` +still checks relation references without prematurely applying native runtime +requirements to partially reconstructed CellProfiler contracts. Real compile +inspection reaches the finalized validation boundary and fails before execution. + +New-case edit count: one owner declaration for a new setting binding, zero new +catalog/compiler branches; the registered/new-binding regression exercises this. +A new subject relation exposes its existing `measurement_subject()` capability; +both compile and runtime use it unchanged. A new output kind supplies its own +validation hook, with zero edits to the generic compiler consumer. The +duplicate-resolution removal is factoring; no pure file move is claimed. ## Remaining acceptance -- Continuous synthetic two-producer compile, exact-selector document roundtrip, - runtime binding, and omitted-selector ambiguity regression for #204. -- Realistic current headless compile-inspection entrypoint for both issues. +- The original paired-channel journey fails **before measurement**, while + secondary consumes `Nuclei`: the exact record has fixed channel 1 scope, + but `RuntimeArtifactInput._matches_execution_scope` compares it with the + secondary source's channel 2 scope and rejects it. Address, axis, and producer + identity match. Reproduce with `pytest -q + tests/integration/test_measurement_declaration_journey.py -k paired --tb=short` + under the serialized validation lock when Euler releases it. Zeno preserves + the reproducer; integration coordinator must assign/coordinate the runtime/input + fix before edits outside this worker's measurement scope. No runtime/input, + source preparation, ROI, or viewer implementation was modified. - Coordinator-authorized actual UI code-document/live installation validation. Source tests do not establish installed readiness. Worker must not install or merge, and must not inspect Euler's blind input or output. -- Correct #204's diagnosis after the complete selector proof; retain its original - observed failure. Both issues remain referenced, not claimed closed here. +- #204's corrected diagnosis is catalog discoverability, retaining the original + failure and acceptance verbatim in `issue204_catalog_diagnosis_20260929.md`. + Same-source runtime selection is proven; paired-channel runtime and actual + GUI interaction remain open, so #204 is not claimed closed. + +Draft PR: https://github.com/OpenHCSDev/openhcs/pull/205. + +## Changed paths + +- `openhcs/agent/dto/functions.py` +- `openhcs/agent/services/function_catalog_service.py` +- `openhcs/core/artifact_key_selection.py` +- `openhcs/core/artifacts.py` +- `openhcs/core/callable_contract.py` +- `openhcs/core/function_patterns.py` +- `openhcs/core/steps/function_runtime.py` +- `tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py` +- `tests/unit/test_function_artifact_outputs.py` +- `tests/integration/test_measurement_declaration_journey.py` +- `docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md` +- `docs/plans/issue204_catalog_diagnosis_20260929.md` +Owned disposable test output: `/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929`, +owner Zeno, purpose tiny synthetic plate and bounded pytest temporary outputs. +Clean after preserving test evidence; no source or saved sessions are stored there. No owned large scratch artifacts have been created. Viewer-bind and ROI materialization implementation surfaces remain with their assigned owners. diff --git a/docs/plans/issue204_catalog_diagnosis_20260929.md b/docs/plans/issue204_catalog_diagnosis_20260929.md new file mode 100644 index 0000000000..792924d73c --- /dev/null +++ b/docs/plans/issue204_catalog_diagnosis_20260929.md @@ -0,0 +1,88 @@ +## Clarified diagnosis — 2026-09-29 + +The exact selector already exists on the CellProfiler declaration. This is a +catalog discoverability defect, not a missing selector mechanism: + +- `MeasureObjectIntensityModule.object_measurement_binding.parameter_name` is + `None`, but `require_parameter_name()` supplies + `select_object_sets_to_measure`; the binding is repeated. +- Author `select_object_sets_to_measure=("Cells",)` in the measurement + `FunctionStep` kwargs, where `Cells` is the exact declared secondary output. + This compile-time selector is consumed by declaration reconstruction. +- `labels` is the runtime payload parameter, not an authorable selector. +- The agent catalog projected the raw optional `parameter_name`, concealing the + existing derived selector. Draft [PR205](https://github.com/OpenHCSDev/openhcs/pull/205) + projects `require_parameter_name()` and explains exact tuple authoring. + +Current source verification uses the real `PipelineDocumentAuthority` and +`FunctionStepDocumentAuthority` render/parse routes followed by +`InProcessCompileInspectionGateway.compile`. With primary `Nuclei` and +secondary `Cells` producers, `("Cells",)` survives both roundtrips and the +compiled scalar `labels` edge points only to secondary source step 1. Omitting +selection still fails closed; a one-producer pipeline still compiles. + +A complete same-source two-producer run additionally proves that the real +orchestrator measures the grown secondary `Cells` region, not primary `Nuclei`. +The current focused suite passed 80 unit cases and five real headless journey +cases; the paired-channel regression below was explicitly deselected, not fixed. + +The paired DNA-channel-1 / actin-channel-2 runtime reproduction exposes a separate +input-scope defect before measurement: the secondary step requests exact +`Nuclei` labels under channel 2's fixed execution scope, while their record +belongs to channel 1. `RuntimeArtifactInput._matches_execution_scope` rejects +that address-matched record. This remains an explicit acceptance blocker; it is +not evidence of a missing selector and is not bypassed with fallback selection. +The integration owner must assign/coordinate that runtime/input change. Actual +GUI interaction and installed-stack verification also remain coordinator-owned. + +## Original observed failure and acceptance (preserved) + +Original title: Allow typed exact object-label selection for scalar CellProfiler measurement steps + +## Problem + +An OpenHCS pipeline that produces both primary and secondary object-label +artifacts cannot add a scalar `measure_object_intensity` step to measure the +secondary cells on the actin channel. Compile inspection rejects the step +before execution: + +``` +Callable 'measure_object_intensity' scalar artifact-fed parameter 'labels' has multiple exact artifact occurrences (IdentifyPrimaryObjects_1_object_labels_1, IdentifySecondaryObjects_2_object_labels_1). +``` + +The ambiguity is correctly rejected, but the public PipelineDocument / UI +code-document route must expose a typed way to select the intended exact +producer for that scalar `labels` input. Supplying `labels` as a function kwarg +is not valid because it is runtime-owned; the UI code-document roundtrip also +removed such a kwarg in the observed H003 diagnostic. This is not a request +to guess the newest artifact or silently pick one. + +## Minimal reproducer + +1. In one complete PipelineDocument, step 0 calls + `identify_primary_objects` on a DNA plane and emits primary labels. +2. Step 1 calls `identify_secondary_objects` on a paired actin plane and + consumes those primary labels, emitting secondary labels. +3. Step 2 calls `measure_object_intensity` on the actin plane to measure the + secondary cells. +4. Compile or call `openhcs_inspect_pipeline_source_artifact_plan`. + +The current result is the multiple-exact-occurrences error above. The two +source planes may be tiny synthetic arrays; no biological input is needed to +reproduce this declaration boundary. + +## Acceptance + +- A declaration-owned typed selector identifies the exact secondary-label + producer for the scalar `labels` parameter, through the headless and UI + code-document routes; roundtrip preserves it. +- Compile plan shows one exact secondary-label artifact input for the + measurement step, and a focused synthetic run measures the secondary + objects rather than primary objects. +- An omitted selector when both are available still fails closed with a + useful ambiguity diagnostic; existing unambiguous pipelines still compile. +- Tests cover compile, code-document roundtrip, and runtime binding without + parallel registries, string-name heuristics, or silent fallback. + +Owner: OpenHCS blind-analysis coordinator; continue in an isolated worktree, +keeping the current H003 blind worker independent of prior trial findings. diff --git a/openhcs/agent/dto/functions.py b/openhcs/agent/dto/functions.py index f75859463a..ff254d1aa1 100644 --- a/openhcs/agent/dto/functions.py +++ b/openhcs/agent/dto/functions.py @@ -231,7 +231,7 @@ class CellProfilerArtifactBindingSummary: direction: str kind: str setting_names: tuple[str, ...] - parameter_name: str | None = None + parameter_name: str runtime_parameter_name: str | None = None repeated: bool = False diff --git a/openhcs/core/artifact_key_selection.py b/openhcs/core/artifact_key_selection.py index fb0d9f0473..ab29316ad7 100644 --- a/openhcs/core/artifact_key_selection.py +++ b/openhcs/core/artifact_key_selection.py @@ -55,11 +55,14 @@ def select_plans( selected.append(plan) return tuple(selected) - def validate_artifact_relation_refs(self, *, owner_name: str) -> None: - output_specs = self.artifact_specs.for_plan_type(ArtifactOutputPlan).specs - for spec in output_specs: + def validate_artifact_output_declarations(self) -> None: + """Validate output kinds when this declaration owns returned payloads.""" + + for spec in self.artifact_specs.for_plan_type(ArtifactOutputPlan).specs: spec.artifact_type.validate_output_declaration(spec) + + def validate_artifact_relation_refs(self, *, owner_name: str) -> None: self.artifact_specs.validate_registered_relation_refs( owner_name=owner_name, - relation_specs=output_specs, + relation_specs=self.artifact_specs.for_plan_type(ArtifactOutputPlan).specs, ) diff --git a/openhcs/core/callable_contract.py b/openhcs/core/callable_contract.py index ef3d48d2a0..cc3bfe1a09 100644 --- a/openhcs/core/callable_contract.py +++ b/openhcs/core/callable_contract.py @@ -1146,6 +1146,14 @@ def bind_runtime_owned(parameter_name: str) -> None: ) from exc return tuple(kwargs.items()) + def validate_artifact_output_declarations(self) -> None: + """Validate native returns, leaving adapter-recorded rows with their owner.""" + + adapter = self.runtime_adapter + if adapter is not None and adapter.manages_artifact_outputs: + return + super(CallableContract, self).validate_artifact_output_declarations() + def validate_artifact_input_parameter_bindings(self) -> None: """Validate exact artifact occurrences against the normalized callable ABI.""" diff --git a/openhcs/core/function_patterns.py b/openhcs/core/function_patterns.py index bb45eadce1..d9e1523044 100644 --- a/openhcs/core/function_patterns.py +++ b/openhcs/core/function_patterns.py @@ -1223,9 +1223,7 @@ def _compile_invocation( ) item = replace(item, contract=contract_plan.contract) artifact_selector = declaration_provider(item, step_context) - artifact_selector.validate_artifact_relation_refs( - owner_name=item.contract.function_name, - ) + artifact_selector.validate_artifact_output_declarations() artifact_input_plans = artifact_selector.select_plans( ArtifactInputPlan, input_plans, diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py new file mode 100644 index 0000000000..4aa56fc199 --- /dev/null +++ b/tests/integration/test_measurement_declaration_journey.py @@ -0,0 +1,344 @@ +"""Real source-document, headless compiler, and runtime selector journey.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pytest +import tifffile + +from openhcs.agent.services.execution_session_service import ( + AgentProgressQueue, + CompileInspectionInput, + InProcessCompileInspectionGateway, +) +from openhcs.constants import AllComponents, GroupBy, Microscope, VariableComponents +from openhcs.constants.input_source import InputSource +from openhcs.core.artifacts import ( + ArtifactInputPlan, + ArtifactOutputPlan, + ArtifactSpec, + ImageArtifactType, + ImageMeasurementSubjectRelation, + MainFlowStackOutputSpec, + MeasurementsArtifactType, + ObjectLabelsArtifactType, +) +from openhcs.core.config import ( + GlobalPipelineConfig, + LazyProcessingConfig, + LazySourceBindingsConfig, + PipelineConfig, +) +from openhcs.core.function_patterns import normalize_function_pattern +from openhcs.core.function_step_document import FunctionStepDocumentAuthority +from openhcs.core.measurement_feature_queries import measurement_values_for_feature +from openhcs.core.measurement_row_materialization import DataclassMeasurementColumnarRows +from openhcs.core.memory import numpy +from openhcs.core.orchestrator.execution_result import RuntimeObservationMode +from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator +from openhcs.core.pipeline_document import PipelineDocumentAuthority +from openhcs.core.pipeline.function_contracts import artifact_outputs +from openhcs.core.runtime_object_labels import object_label_dense_array +from openhcs.core.source_bindings import ( + ComponentSelector, + NamedSourceBinding, + SourceFilterClause, + SourceFilterMatchType, + SourceFilterSubject, + SourceSelector, + StepSourceBindingsConfig, +) +from openhcs.core.steps.function_step import FunctionStep +from openhcs.interop.cellprofiler.measurement_dialect import ( + CELLPROFILER_MEASUREMENT_LOOKUP_DIALECT, +) +from openhcs.processing.backends.cellprofiler.intensity import ( + MeasureObjectIntensityModule, + measure_object_intensity, +) +from openhcs.processing.backends.cellprofiler.primary_objects import ( + IdentifyPrimaryObjectsModule, + UnclumpMethod, + WatershedMethod, + identify_primary_objects, +) +from openhcs.processing.backends.cellprofiler.secondary import ( + IdentifySecondaryObjectsModule, + SecondaryMethod, + identify_secondary_objects, +) +from openhcs.processing.backends.cellprofiler.thresholding import ( + CellProfilerThresholdMethod, +) +from openhcs.processing.backends.lib_registry.unified_registry import ProcessingContract +from openhcs.processing.materialization import CsvOptions, MaterializationSpec + + +@dataclass(frozen=True) +class CountRow: + slice_index: int + pixel_count: int + + +COUNT_IMAGE = MainFlowStackOutputSpec.output("CountedImage", ImageArtifactType) +MISSING_SUBJECT_ROWS = ArtifactSpec.output( + "PixelCounts", MeasurementsArtifactType, + materialization=MaterializationSpec(CsvOptions()), +) +IMAGE_SUBJECT_ROWS = ArtifactSpec.output( + "PixelCounts", MeasurementsArtifactType, + materialization=MaterializationSpec(CsvOptions()), + relations=(ImageMeasurementSubjectRelation(COUNT_IMAGE.ref()),), +) + + +@numpy(contract=ProcessingContract.PURE_3D) +@artifact_outputs(COUNT_IMAGE, MISSING_SUBJECT_ROWS) +def count_without_subject(image): + raise AssertionError("A missing subject must fail before callable execution") + + +@numpy(contract=ProcessingContract.PURE_3D) +@artifact_outputs(COUNT_IMAGE, IMAGE_SUBJECT_ROWS) +def count_with_image_subject(image): + return image, DataclassMeasurementColumnarRows( + (CountRow(0, int(np.count_nonzero(image))),), row_type=CountRow + ) + + +def _binding_name(module, plan_type, artifact_type): + (binding,) = module.declared_artifact_bindings( + plan_type=plan_type, artifact_type=artifact_type + ) + return binding.require_parameter_name() + + +def _source(alias, channel): + return NamedSourceBinding( + alias=alias, + selector=SourceSelector( + filters=( + SourceFilterClause( + subject=SourceFilterSubject.FILE, + match_type=SourceFilterMatchType.CONTAINS, + value=alias, + ), + ), + ), + component_identity=(ComponentSelector(AllComponents.CHANNEL, channel),), + ) + + +def _step(func, name, kwargs): + return FunctionStep( + func=(func, kwargs), + name=name, + processing_config=LazyProcessingConfig( + variable_components=[VariableComponents.SITE], + group_by=GroupBy.NONE, + input_source=InputSource.PIPELINE_START, + ), + source_bindings=StepSourceBindingsConfig(enabled=True), + ) + + +def _document(*, selected=True, two_producers=True, same_source=False): + measurement_image = "DNA" if same_source else "Actin" + primary = _step( + identify_primary_objects, + "Primary", + { + _binding_name(IdentifyPrimaryObjectsModule, ArtifactInputPlan, ImageArtifactType): "DNA", + _binding_name(IdentifyPrimaryObjectsModule, ArtifactOutputPlan, ObjectLabelsArtifactType): "Nuclei", + "exclude_size": False, + "exclude_border_objects": False, + "unclump_method": UnclumpMethod.NONE, + "watershed_method": WatershedMethod.NONE, + "threshold_method": CellProfilerThresholdMethod.MANUAL, + "manual_threshold": 0.5, + "threshold_smoothing_scale": 0.0, + }, + ) + secondary = _step( + identify_secondary_objects, + "Secondary", + { + _binding_name(IdentifySecondaryObjectsModule, ArtifactInputPlan, ImageArtifactType): measurement_image, + _binding_name(IdentifySecondaryObjectsModule, ArtifactInputPlan, ObjectLabelsArtifactType): "Nuclei", + IdentifySecondaryObjectsModule.secondary_output_binding.require_parameter_name(): "Cells", + "method": SecondaryMethod.DISTANCE_N, + "distance_to_dilate": 2, + }, + ) + kwargs = { + MeasureObjectIntensityModule.image_measurement_binding.require_parameter_name(): (measurement_image,), + } + if selected: + kwargs[ + MeasureObjectIntensityModule.object_measurement_binding.require_parameter_name() + ] = ("Cells" if two_producers else "Nuclei",) + measure = _step(measure_object_intensity, "Measure cells", kwargs) + return PipelineDocumentAuthority.from_values( + pipeline_config=PipelineConfig( + microscope=Microscope.SOURCE_BINDINGS, + source_bindings_config=LazySourceBindingsConfig( + bindings=(_source("DNA", "1"),) if same_source else ( + _source("DNA", "1"), _source("Actin", "2"), + ), + ), + ), + pipeline_steps=[primary, secondary, measure] if two_producers else [primary, measure], + ) + + +def _write_plate(path, *, same_source=False): + dna = np.full((16, 16), 0.25 if same_source else 0.0, dtype=np.float32) + dna[6:8, 6:8] = 1.0 + actin = np.ones_like(dna) + tifffile.imwrite(path / "A01_s001_DNA.tif", dna) + tifffile.imwrite(path / "A01_s001_Actin.tif", actin) + + +def _compile(path, document, global_config): + return InProcessCompileInspectionGateway().compile( + CompileInspectionInput( + plate=path, + pipeline_document=document, + axis_filter=("A01",), + global_pipeline_config=global_config, + progress_queue=AgentProgressQueue(), + ) + ).execution_bundle + + +def _execute(path, document, bundle): + orchestrator = PipelineOrchestrator( + path, pipeline_config=document.pipeline_config + ).initialize() + return orchestrator.execute_compiled_plate( + execution_bundle=bundle, + max_workers=1, + runtime_observation_mode=RuntimeObservationMode.MERGE_INTO_PARENT, + progress_queue=AgentProgressQueue(), + progress_context={ + "execution_id": "measurement-contract-synthetic", + "plate_id": str(path), + "axis_id": "", + }, + ) + + +@pytest.mark.parametrize("same_source", [True, False], ids=["single-source", "paired-channels"]) +def test_exact_secondary_selector_survives_authoring_compile_and_execution(tmp_path, same_source): + _write_plate(tmp_path, same_source=same_source) + document = _document(same_source=same_source) + selector = MeasureObjectIntensityModule.object_measurement_binding.require_parameter_name() + source = PipelineDocumentAuthority.render(document) + reconstructed = PipelineDocumentAuthority.from_source(source) + measurement = reconstructed.pipeline_steps[-1] + step_source = FunctionStepDocumentAuthority.render( + FunctionStepDocumentAuthority.from_value(measurement) + ) + measurement = FunctionStepDocumentAuthority.from_source(step_source).step + reconstructed.pipeline_steps[-1] = measurement + authored = next(normalize_function_pattern(measurement.func).iter_items()) + assert authored.kwargs_dict[selector] == ("Cells",) + assert "labels" not in authored.kwargs_dict + + global_config = GlobalPipelineConfig(num_workers=1, use_threading=True) + bundle = _compile(tmp_path, reconstructed, global_config) + context = bundle.runtime_contexts["A01"] + plan = context.step_plans[2] + invocation = next(plan.compiled_function_pattern.iter_invocations()) + (edge,) = ( + edge for edge in invocation.artifact_input_edges + if edge.spec.artifact_type is ObjectLabelsArtifactType + ) + assert edge.spec.name == "Cells" + assert edge.spec.parameter_name == "labels" + assert edge.storage_plan.source_step_id == 1 + assert selector not in invocation.kwargs_dict + assert "labels" not in invocation.kwargs_dict + + results = _execute(tmp_path, reconstructed, bundle) + assert results["A01"].is_success(), results["A01"].error_message + store = context.runtime_value_store + [primary] = store.find(name="Nuclei", axis_id="A01") + [secondary] = store.find(name="Cells", axis_id="A01") + primary_area = np.count_nonzero(object_label_dense_array(primary.value.data)) + secondary_area = np.count_nonzero(object_label_dense_array(secondary.value.data)) + assert secondary_area > primary_area > 0 + [output] = invocation.artifact_output_plans + [measurement] = store.find(name=output.name, artifact_type=MeasurementsArtifactType, axis_id="A01") + assert measurement.value.data.subject.object_name == "Cells" + image_name = "DNA" if same_source else "Actin" + values = measurement_values_for_feature( + (measurement.value.data,), + f"Intensity_IntegratedIntensity_{image_name}", + object_count=1, + object_name="Cells", + dialect=CELLPROFILER_MEASUREMENT_LOOKUP_DIALECT, + ) + # The DNA seed is four bright pixels; the grown cell adds dim background. + # Measuring primary labels instead would return exactly 4.0. + expected = 4.0 + 0.25 * (secondary_area - 4) if same_source else float(secondary_area) + assert tuple(values) == (expected,) + + +def test_omitted_secondary_selector_still_fails_closed(tmp_path): + _write_plate(tmp_path) + with pytest.raises(ValueError, match="labels.*multiple exact artifact occurrences"): + _compile( + tmp_path, + _document(selected=False), + GlobalPipelineConfig(num_workers=1, use_threading=True), + ) + + +def test_omitted_selector_remains_valid_for_one_label_producer(tmp_path): + _write_plate(tmp_path) + bundle = _compile( + tmp_path, + _document(selected=False, two_producers=False), + GlobalPipelineConfig(num_workers=1, use_threading=True), + ) + invocation = next( + bundle.runtime_contexts["A01"].step_plans[1].compiled_function_pattern.iter_invocations() + ) + (edge,) = ( + edge for edge in invocation.artifact_input_edges + if edge.spec.artifact_type is ObjectLabelsArtifactType + ) + assert edge.spec.name == "Nuclei" + + +@pytest.mark.parametrize("valid", [False, True]) +def test_headless_entrypoint_requires_subject_and_executes_corrected_rows(tmp_path, valid): + _write_plate(tmp_path) + document = PipelineDocumentAuthority.from_values( + pipeline_config=PipelineConfig( + microscope=Microscope.SOURCE_BINDINGS, + source_bindings_config=LazySourceBindingsConfig( + bindings=(_source("DNA", "1"),), + ), + ), + pipeline_steps=[_step( + count_with_image_subject if valid else count_without_subject, + "Count pixels", {}, + )], + ) + global_config = GlobalPipelineConfig(num_workers=1, use_threading=True) + if not valid: + with pytest.raises(ValueError, match="PixelCounts.*no declared measurement subject") as exc: + _compile(tmp_path, document, global_config) + assert "ImageMeasurementSubjectRelation" in str(exc.value) + return + bundle = _compile(tmp_path, document, global_config) + results = _execute(tmp_path, document, bundle) + assert results["A01"].is_success(), results["A01"].error_message + store = bundle.runtime_contexts["A01"].runtime_value_store + [counts] = store.find(name="PixelCounts", axis_id="A01") + assert counts.value.data.subject.source_image_name == "CountedImage" + assert tuple(counts.value.data.rows.column_values("pixel_count")) == (4,) diff --git a/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py b/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py index 729d931959..b8994c015a 100644 --- a/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py +++ b/tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py @@ -16,6 +16,9 @@ MeasureObjectIntensityModule, measure_object_intensity, ) +from openhcs.interop.cellprofiler.module_declarations import CellProfilerModule +from openhcs.interop.cellprofiler.settings_binder import SettingToKeywordBinding +from openhcs.core.artifacts import ObjectLabelsArtifactType @dataclass(frozen=True) @@ -189,3 +192,35 @@ def test_catalog_exposes_declared_exact_object_selector_and_authoring_guidance( assert "do not pass labels" in contract.source_binding_rule parameters = {parameter.name: parameter for parameter in detail.parameters} assert parameters["labels"].supplied_by is FunctionParameterSource.ARTIFACT_INPUT + + +def test_catalog_projects_registered_and_new_binding_declarations_without_a_roster(): + from openhcs.agent.services.function_catalog_service import ( + _cellprofiler_artifact_binding_summary, + ) + + registered = tuple( + binding + for module in CellProfilerModule.__registry__.values() + for binding in module.declared_artifact_bindings() + ) + assert registered + new_bindings = ( + SettingToKeywordBinding.input( + "Select future object cohort", + ObjectLabelsArtifactType, + runtime_parameter_name="labels", + repeated=True, + ), + SettingToKeywordBinding.output( + "Name future object cohort", + ObjectLabelsArtifactType, + parameter_name="exact_future_output", + ), + ) + for binding in (*registered, *new_bindings): + summary = _cellprofiler_artifact_binding_summary(binding) + assert summary.parameter_name == binding.require_parameter_name() + assert summary.runtime_parameter_name == binding.runtime_parameter_name + assert summary.repeated is binding.repeated + assert summary.kind == binding.require_artifact_type().require_value() diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index 55d53509e0..3d9f46d9de 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -2320,7 +2320,7 @@ class _NativeCountRow: cell_count: int -@pytest.mark.parametrize("boundary", ["inspection", "compile"]) +@pytest.mark.parametrize("boundary", ["contract", "compile"]) def test_measurement_subject_is_required_before_schema_rows_execute(boundary): spec = ArtifactSpec.output( "cell_counts", @@ -2333,8 +2333,10 @@ def count(image): raise AssertionError("Invalid measurement declarations must not execute") with pytest.raises(ValueError, match="cell_counts.*no declared measurement subject") as exc: - if boundary == "inspection": - extract_artifact_declarations(count) + if boundary == "contract": + from openhcs.core.callable_contract import CallableContract + + CallableContract.from_callable(count).validate_artifact_output_declarations() else: compile_function_pattern(count, {}, {}) assert "ImageMeasurementSubjectRelation" in str(exc.value) @@ -2760,7 +2762,11 @@ def test_execute_function_core_requires_all_declared_artifact_values(): context = ContextStub() @artifact_outputs( - ArtifactSpec.output("measurements", MeasurementsArtifactType), + ArtifactSpec.output( + "measurements", + MeasurementsArtifactType, + relations=(ArtifactMeasurementSubjectRelation(),), + ), ) def analyze(image): return (image,) From 688e4e7d88f163d3f21c094ce4a599365dd3faa1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 14:48:31 -0400 Subject: [PATCH 03/17] Record owned synthetic validation scratch cleanup --- ...ssue133_204_measurement_contract_checkpoint_20260929.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 7d5847113a..44d17bef5f 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -133,8 +133,9 @@ Draft PR: https://github.com/OpenHCSDev/openhcs/pull/205. - `docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md` - `docs/plans/issue204_catalog_diagnosis_20260929.md` -Owned disposable test output: `/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929`, +Owned disposable test output was `/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929`, owner Zeno, purpose tiny synthetic plate and bounded pytest temporary outputs. -Clean after preserving test evidence; no source or saved sessions are stored there. -No owned large scratch artifacts have been created. Viewer-bind and ROI +Removed its 608 KiB after preserving evidence and the fully self-seeding source +reproducer. Generated arrays and logs are regeneratable; no source or saved +sessions were stored there. No owned large scratch artifacts were created. Viewer-bind and ROI materialization implementation surfaces remain with their assigned owners. From b64d9aef9c0eeacc06d60bdf0d53ab27837a44c0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 15:18:16 -0400 Subject: [PATCH 04/17] Resolve declared secondary artifact inputs by typed source context --- ...easurement_contract_checkpoint_20260929.md | 77 +++++- .../issue204_catalog_diagnosis_20260929.md | 11 +- openhcs/core/component_group_scope.py | 25 +- openhcs/core/runtime_stores.py | 49 ++-- openhcs/core/steps/function_runtime.py | 1 + .../interop/cellprofiler/runtime/adapter.py | 1 + .../cellprofiler/runtime/artifact_binding.py | 1 + tests/unit/test_runtime_value_store.py | 233 ++++++++++++++++++ 8 files changed, 371 insertions(+), 27 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 44d17bef5f..de30be99be 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -2,9 +2,10 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. -Base: `openhcsdev/main` at `98d9b9d23`, fetched and merged normally on 2026-09-29. -Fetched again before publishing the regression checkpoint; normal merge reported -already up to date. No parent/integration edits, rebases, resets, or gitlink changes. +Base: `openhcsdev/main` at `a0263e82a1`, fetched and merged normally on 2026-09-29. +The earlier checkpoint used `98d9b9d23`; the runtime checkpoint integrates new +main through normal merge `9f5d444515`. No parent/main-worktree edits, rebases, +resets, installed-source edits, or gitlink changes. Closes #133. References #204; its paired-channel and actual GUI acceptance remain open. This is a source-verified draft, not an installed/live-readiness claim. @@ -25,6 +26,16 @@ open. This is a source-verified draft, not an installed/live-readiness claim. discoverable separately from the runtime-owned scalar `labels` parameter. - Catalog guidance explains exact-name tuples and consumed compile-time selectors. No selector registry, artifact store, fallback, or ambiguity relaxation was added. +- The secondary module already declares `InputGroupLineageSourceRelation` from its + primary-label input to its image input. Runtime input matching now projects typed + scope identities using that exact relation's source bindings and the existing + `SourceImageSetIdentityPolicy` / `SourceImageSetIdentityCompatibility` owners. + This permits declared paired-channel context without a global channel exception. + Missing relations or unrelated visible bindings retain exact coordinate checks. +- All three production input constructors forward their existing compiled source + binding plan: native callable loading, adapter record access, and binding requests. + Producer address/group selection, stack-plane projection, and ambiguity checks + remain authoritative; the old duplicate fixed-coordinate loops are deleted. ## Focused evidence @@ -51,7 +62,7 @@ their recorded gitlinks without installed-source changes. retrieval uses the existing CellProfiler measurement dialect. - The paired DNA/actin-channel case also passes roundtrip and compile, but its earlier continuous run failed at the secondary step's primary-label input. - It remains an unweakened failing regression, deselected explicitly above; + It remains an unweakened regression, deselected in that historical run; see the remaining acceptance section. No xfail or fallback was added. - Omitted selection with two producers fails closed with the original useful ambiguity diagnostic; omitted selection with one producer compiles. @@ -65,11 +76,29 @@ their recorded gitlinks without installed-source changes. The final run completed and released the lock before Euler's H003 restart; no further heavy validation was started after the resource-priority instruction. +Runtime checkpoint after the explicit scope extension: + +- `pytest -q tests/unit/test_runtime_value_store.py --tb=short`: **48 passed**, + 1.53 seconds, provider-free and without a JVM, GUI, or MCP server. New cases + exercise paired-channel matching, both adapter consumers, conflicting fixed + site/z/time coordinates, another well or exact producer path, missing source + declarations, missing context relations, a relation to an unrelated source, + an absent producer site plane, and unresolved multiple-record ambiguity. +- Replaying the published matching body on the same new fixture rejects channel + 1 versus 2; the new body selects its exact producer. This is provider-free + reproduction of the precise owner fault, not complete paired execution. +- Attempting the complete six-case headless journey under nonblocking `flock -n + -E 75` returned **75 (lock busy)**, without starting pytest. Euler's H003 slot + remains reserved. Paired runtime acceptance is pending this run, not claimed + from the 48 focused store checks. No deselection is planned for it. +- The resource guard reported 11.6 GiB available RAM; its warning was historical + 11.5 GiB used swap. Only lightweight source/unit checks proceeded. + ## Essential anti-pattern audit: actual ownership evidence This is a focused source/caller and AST audit against the current archived refactor-audit pattern catalog, **not a complete NRA semantic/proof scan**. All -seven changed production files were parsed against the merged base: zero added +eleven changed production files were parsed against current merged main: zero added `match`, `isinstance`, `getattr`/`hasattr`, or 6+-term boolean-chain AST nodes. That guard detects listed shapes, not every possible semantic ownership defect. @@ -80,6 +109,7 @@ That guard detects listed shapes, not every possible semantic ownership defect. | IDEN-1/2/5, BOUND-2/7, MEMB-3/4 | `SettingToKeywordBinding.require_parameter_name()` owns selector identity, separately from `runtime_parameter_name`. The only production `CellProfilerArtifactBindingSummary` constructor projects that owner; its selector field is now required. No selector store, guessed names, raw-row decoder, or attribute-name probe exists in the change. The catalog regression derives every registered binding and exercises new derived-name input and explicit-name output declarations without editing the projection. | | IDEN-7, IMPL-10 | An initial placement incorrectly rejected heterogeneous CellProfiler records. Validation now runs after invocation-contract finalization, where the existing `runtime_adapter.manages_artifact_outputs` capability names the output owner. Native table-wide subject invariants apply only to native returned payloads, not adapter-recorded tables. Real primary/secondary compilation and execution plus native missing/corrected tests exercise both owners. | | AGENT-2/3/6/7, TIME-5/9 | No second mechanism, format converter, test-only production switch, or dormant feature was added. One pre-existing cardinality fixture received a valid artifact-level subject so its original missing-return assertion remains meaningful. The paired-channel failure is retained, not hidden by weaker expectations; publication proceeds with that exact acceptance gap stated. | +| IDEN-1/6/7, BOUND-2, MEMB-1/2/3, IMPL-5/12, TIME-1/3 | Exact compiled producer/address selection and contextual source compatibility answer different questions. `RuntimeArtifactInput` derives context membership from `edge.spec.source_context_sources()` and the referenced original source bindings, then delegates matching to the existing typed identity predicate. `RuntimeExecutionAxisScope` owns its typed-coordinate projection. No literal channel/site/z/time matching roster, new registry, global ignore flag, alternate reader, or copied predicate was added. All three constructors are migrated; two actual adapter consumers and fail-closed new cases execute. The replaced fixed-coordinate loops are gone. | Caller review covered the default invocation declaration provider (returns the finalized `CallableContract`), every call to the new output-validation hook and @@ -89,6 +119,21 @@ still checks relation references without prematurely applying native runtime requirements to partially reconstructed CellProfiler contracts. Real compile inspection reaches the finalized validation boundary and fails before execution. +Runtime caller review also covered `records`, static/dynamic `all_records`, +`_records`, `_validate_selected_records`, `_axis_value`, `projected_values`, +`composed_value`, all three production `RuntimeArtifactInput` constructors, and +the existing secondary input relation declaration. Compiled group selection and +variable-component projection are unchanged: existing complete-producer-group +and transposed-stack tests still pass, while the new missing-site-plane test fails +closed. This checks this runtime surface; it is not a global provenance audit. + +New-case runtime edit count: a context relation and a source declaration supply +membership, with zero changes to the matching consumer or a component roster. +Tests show a visible binding cannot substitute for a missing relation, and a +relation cannot borrow membership from an unrelated source. Partially scoped +identities continue using the existing compatibility predicate, not a new +equality algorithm. + New-case edit count: one owner declaration for a new setting binding, zero new catalog/compiler branches; the registered/new-binding regression exercises this. A new subject relation exposes its existing `measurement_subject()` capability; @@ -98,16 +143,23 @@ duplicate-resolution removal is factoring; no pure file move is claimed. ## Remaining acceptance -- The original paired-channel journey fails **before measurement**, while +- The original paired-channel journey failed **before measurement**, while secondary consumes `Nuclei`: the exact record has fixed channel 1 scope, but `RuntimeArtifactInput._matches_execution_scope` compares it with the secondary source's channel 2 scope and rejects it. Address, axis, and producer identity match. Reproduce with `pytest -q tests/integration/test_measurement_declaration_journey.py -k paired --tb=short` - under the serialized validation lock when Euler releases it. Zeno preserves - the reproducer; integration coordinator must assign/coordinate the runtime/input - fix before edits outside this worker's measurement scope. No runtime/input, - source preparation, ROI, or viewer implementation was modified. + under the serialized validation lock when Euler releases it. Zeno is now the + active runtime-fix owner under the explicit scope extension. The focused fix is + present and store regressions pass, but complete paired execution remains + unverified. No `core/source_bindings.py`, `core/source_binding_workspace.py`, + source preparation, ROI, viewer, or #211 authorization method/test was modified. +- Source-only explanation of the apparent grouped/ungrouped distinction: group + coordinates are selected by the compiled edge, while the previous predicate + compared fixed coordinates on either side. Binding-owned channel identities + expose this failure with `GroupBy.NONE`; a channel selected as a producer group + takes another existing path. No Euler data, output, pipeline, or run + configuration was opened to infer its particular success mechanism. - Coordinator-authorized actual UI code-document/live installation validation. Source tests do not establish installed readiness. Worker must not install or merge, and must not inspect Euler's blind input or output. @@ -126,9 +178,14 @@ Draft PR: https://github.com/OpenHCSDev/openhcs/pull/205. - `openhcs/core/artifacts.py` - `openhcs/core/callable_contract.py` - `openhcs/core/function_patterns.py` +- `openhcs/core/component_group_scope.py` +- `openhcs/core/runtime_stores.py` - `openhcs/core/steps/function_runtime.py` +- `openhcs/interop/cellprofiler/runtime/adapter.py` +- `openhcs/interop/cellprofiler/runtime/artifact_binding.py` - `tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py` - `tests/unit/test_function_artifact_outputs.py` +- `tests/unit/test_runtime_value_store.py` - `tests/integration/test_measurement_declaration_journey.py` - `docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md` - `docs/plans/issue204_catalog_diagnosis_20260929.md` diff --git a/docs/plans/issue204_catalog_diagnosis_20260929.md b/docs/plans/issue204_catalog_diagnosis_20260929.md index 792924d73c..d69ee0484d 100644 --- a/docs/plans/issue204_catalog_diagnosis_20260929.md +++ b/docs/plans/issue204_catalog_diagnosis_20260929.md @@ -32,8 +32,15 @@ input-scope defect before measurement: the secondary step requests exact belongs to channel 1. `RuntimeArtifactInput._matches_execution_scope` rejects that address-matched record. This remains an explicit acceptance blocker; it is not evidence of a missing selector and is not bypassed with fallback selection. -The integration owner must assign/coordinate that runtime/input change. Actual -GUI interaction and installed-stack verification also remain coordinator-owned. +Zeno now owns that runtime/input change under the explicit scope extension. PR205 +reuses the existing secondary input's `InputGroupLineageSourceRelation` and +declaration-derived typed image-set identity policy, not a global channel-ignore +flag or another selector. All 48 lightweight runtime-store regressions pass, +including both adapter consumers and wrong well/site/z/time/producer, missing +relation/source, wrong referenced source, absent site plane, and ambiguity cases. +The full paired headless journey could not acquire Euler's validation lock +(exit 75, no pytest started) and remains pending without deselection. Actual GUI +interaction and installed-stack verification also remain coordinator-owned. ## Original observed failure and acceptance (preserved) diff --git a/openhcs/core/component_group_scope.py b/openhcs/core/component_group_scope.py index 23a9104b8e..999aef0bd7 100644 --- a/openhcs/core/component_group_scope.py +++ b/openhcs/core/component_group_scope.py @@ -17,7 +17,11 @@ if TYPE_CHECKING: from openhcs.core.context.processing_context import ProcessingContext - from openhcs.core.source_matching import SourceAxisMetadataScope + from openhcs.core.source_matching import ( + SourceAxisMetadataScope, + SourceImageSetIdentity, + SourceImageSetIdentityPolicy, + ) ComponentGroupKey = str | None RuntimeFixedComponentValues = tuple[tuple[AllComponents, str], ...] @@ -529,6 +533,25 @@ def source_axis_metadata_scope(self) -> "SourceAxisMetadataScope": ) return SourceAxisMetadataScope.from_component_values(component_values) + def source_image_set_identity( + self, + policy: "SourceImageSetIdentityPolicy", + *, + components: ComponentSet, + ) -> "SourceImageSetIdentity": + """Project typed coordinates through declared image-set membership.""" + + from openhcs.core.source_matching import SourceImageSetIdentity + + return SourceImageSetIdentity( + tuple( + (component.value, value) + for component, value in self.source_component_values + if component in components + if policy.is_identity_component(component) + ) + ) + def matching_component_plane_indices( self, component_metadata: Sequence[Mapping[str, object] | None], diff --git a/openhcs/core/runtime_stores.py b/openhcs/core/runtime_stores.py index ef2bb14214..b22fec0459 100644 --- a/openhcs/core/runtime_stores.py +++ b/openhcs/core/runtime_stores.py @@ -41,6 +41,7 @@ from openhcs.core.runtime_slice_projection import RuntimeSliceProjection from openhcs.core.source_matching import ( SourceAxisMetadataScope, + SourceImageSetIdentityCompatibility, SourceImageSetIdentityPolicy, ) from openhcs.core.source_bindings import CompiledSourceBindingPlan @@ -259,6 +260,9 @@ class RuntimeArtifactInput: edge_plan: InvocationArtifactInputEdgePlan axis_scope: RuntimeExecutionAxisScope backend: str + source_binding_plan: CompiledSourceBindingPlan = dataclass_field( + default_factory=CompiledSourceBindingPlan.empty + ) def __post_init__(self) -> None: if not isinstance(self.edge_plan, InvocationArtifactInputEdgePlan): @@ -554,7 +558,7 @@ def _matches_execution_scope( self, record: StoredRuntimeValue, ) -> bool: - """Match fixed consumer coordinates not carried on producer payload axes.""" + """Match image-set context, preserving declared producer/axis projections.""" projected_components = ComponentSet.coerce( self.edge_plan.projection.projected_variable_components( @@ -562,19 +566,36 @@ def _matches_execution_scope( ) ) record_scope = record.key.scope - for component, value in self.axis_scope.fixed_component_values: - if component in projected_components: - continue - producer_value = record_scope.value_text_for_component(component) - if producer_value is not None and producer_value != value: - return False - for component, value in record_scope.fixed_component_values: - if component in projected_components: - continue - consumer_value = self.axis_scope.value_text_for_component(component) - if consumer_value is not None and consumer_value != value: - return False - return True + # Producer groups are selected above by the exact compiled edge. Fixed + # coordinates constrain context on either side; projected stack axes are + # selected from payload provenance by _axis_value, not by record scope. + context_components = ComponentSet.collect( + (component for component, _value in self.axis_scope.fixed_component_values), + (component for component, _value in record_scope.fixed_component_values), + ( + component + for component in AllComponents + if component.is_multiprocessing_axis() + ), + ).excluding(projected_components) + # Only a declared source-context relation supplies plane membership. + # Unrelated visible bindings must not relax an exact artifact input. + context_sources = self.edge_plan.spec.source_context_sources() + identity_policy = ( + SourceImageSetIdentityPolicy.from_source_bindings( + self.source_binding_plan.for_artifact_refs(context_sources) + ) + if context_sources + else SourceImageSetIdentityPolicy() + ) + return SourceImageSetIdentityCompatibility( + record_scope.source_image_set_identity( + identity_policy, components=context_components, + ), + self.axis_scope.source_image_set_identity( + identity_policy, components=context_components, + ), + ).matches() @dataclass(frozen=True, slots=True) diff --git a/openhcs/core/steps/function_runtime.py b/openhcs/core/steps/function_runtime.py index 4b33f735c5..7362c8901f 100644 --- a/openhcs/core/steps/function_runtime.py +++ b/openhcs/core/steps/function_runtime.py @@ -1669,6 +1669,7 @@ def _load_artifact_input_values( edge_plan=input_plan, axis_scope=runtime_scope.axis_scope, backend=Backend.MEMORY.value, + source_binding_plan=runtime_scope.source_binding_plan, ).projected_values(context.runtime_value_store) diff --git a/openhcs/interop/cellprofiler/runtime/adapter.py b/openhcs/interop/cellprofiler/runtime/adapter.py index 55c7d24338..21592a7d79 100644 --- a/openhcs/interop/cellprofiler/runtime/adapter.py +++ b/openhcs/interop/cellprofiler/runtime/adapter.py @@ -139,6 +139,7 @@ def _artifact_input( edge_plan=edge_plan, axis_scope=self.request.axis_scope, backend=self.backend, + source_binding_plan=self.request.source_binding_plan, ) def artifact_input_records( diff --git a/openhcs/interop/cellprofiler/runtime/artifact_binding.py b/openhcs/interop/cellprofiler/runtime/artifact_binding.py index 612cbfdc81..e1926ad220 100644 --- a/openhcs/interop/cellprofiler/runtime/artifact_binding.py +++ b/openhcs/interop/cellprofiler/runtime/artifact_binding.py @@ -658,6 +658,7 @@ def artifact_request( edge_plan=runtime_edge, axis_scope=self.adapter.request.axis_scope, backend=self.adapter.backend, + source_binding_plan=self.adapter.request.source_binding_plan, ) value = cast( RuntimeCallableArgument, diff --git a/tests/unit/test_runtime_value_store.py b/tests/unit/test_runtime_value_store.py index bef6456372..a4dec14e49 100644 --- a/tests/unit/test_runtime_value_store.py +++ b/tests/unit/test_runtime_value_store.py @@ -1,3 +1,5 @@ +from dataclasses import replace + import numpy as np import pytest @@ -8,6 +10,7 @@ ArtifactOutputPlan, ArtifactSpec, ImageArtifactType, + InputGroupLineageSourceRelation, ObjectLabelsArtifactType, MeasurementsArtifactType, ) @@ -29,6 +32,7 @@ ) from openhcs.core.component_group_scope import ComponentGroupScope from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.callable_contract import CallableContract from openhcs.core.runtime_equivalence import ( RuntimeMeasurementObservationAxis, ) @@ -55,6 +59,14 @@ ) from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.core.runtime_artifact_values import ArtifactKey, RuntimeValue +from openhcs.core.runtime_object_labels import ObjectLabelSet, ObjectLabelVariantData +from openhcs.core.source_bindings import ( + CompiledSourceBindingPlan, + ComponentSelector, + NamedSourceBinding, +) +from openhcs.interop.cellprofiler.runtime.artifact_binding import RuntimeInputBindingRequest +from tests.unit.cellprofiler_runtime_test_support import cellprofiler_runtime_adapter_for_test def _runtime_input_edge( @@ -1339,6 +1351,7 @@ def _ungrouped_runtime_artifact_input( storage_plan: ArtifactInputPlan, *, axis_scope: RuntimeExecutionAxisScope = RuntimeExecutionAxisScope(axis_id="A01"), + source_binding_plan: CompiledSourceBindingPlan = CompiledSourceBindingPlan.empty(), ) -> RuntimeArtifactInput: ungrouped = ComponentGroupScope.ungrouped() return RuntimeArtifactInput( @@ -1351,7 +1364,227 @@ def _ungrouped_runtime_artifact_input( ), axis_scope=axis_scope, backend="memory", + source_binding_plan=source_binding_plan, + ) + + +def _paired_channel_label_input(*, producer_axis="A01", producer_values=(), producer_path=None): + """One exact DNA producer consumed in its paired actin image-set context.""" + + fixed_values = { + AllComponents.CHANNEL: "1", + AllComponents.SITE: "1", + AllComponents.Z_INDEX: "1", + AllComponents.TIMEPOINT: "1", + **dict(producer_values), + } + storage_plan = ArtifactInputPlan( + name="Nuclei", + path="/memory/primary/Nuclei.pkl", + artifact_type=ObjectLabelsArtifactType, + source_step_id=0, + ) + output_plan = ArtifactOutputPlan( + name=storage_plan.name, + path=producer_path or storage_plan.path, + artifact_type=storage_plan.artifact_type, + ) + value = RuntimeValue.normalize_for_execution_scope( + output_plan, + ObjectLabelSet( + name=storage_plan.name, + variant_data=ObjectLabelVariantData(labels=np.ones((2, 2), dtype=np.uint16)), + ), + execution_scope=RuntimeExecutionAxisScope.from_raw( + producer_axis, + component=None, + value=None, + fixed_component_values=tuple(fixed_values.items()), + ), ) + store = RuntimeValueStore() + record = store.record(value, path=output_plan.path, backend="memory") + consumer_scope = RuntimeExecutionAxisScope.from_raw( + "A01", + component=None, + value=None, + fixed_component_values=( + (AllComponents.CHANNEL, "2"), + (AllComponents.SITE, "1"), + (AllComponents.Z_INDEX, "1"), + (AllComponents.TIMEPOINT, "1"), + ), + ) + source_bindings = CompiledSourceBindingPlan( + bindings=(NamedSourceBinding( + alias="Actin", + component_identity=(ComponentSelector(AllComponents.CHANNEL, "2"),), + ),), + ) + runtime_input = _ungrouped_runtime_artifact_input( + storage_plan, + axis_scope=consumer_scope, + source_binding_plan=source_bindings, + ) + declared_spec = runtime_input.edge_plan.spec.with_group_scope_relation( + InputGroupLineageSourceRelation( + source=source_bindings.binding_declarations[0].input_spec().ref() + ) + ) + return store, record, replace( + runtime_input, + edge_plan=replace(runtime_input.edge_plan, spec=declared_spec), + ) + + +def test_declared_paired_channel_label_input_matches_image_set_context(): + store, record, runtime_input = _paired_channel_label_input() + + assert runtime_input.records(store) == (record,) + assert runtime_input.resolve_value(store).name == "Nuclei" + + +def test_paired_channel_declaration_reaches_both_adapter_input_consumers(): + store, record, runtime_input = _paired_channel_label_input() + edge = runtime_input.edge_plan + + def consume_labels(image, labels): + return image + + (source_binding,) = runtime_input.source_binding_plan.binding_declarations + source_spec = source_binding.input_spec() + source_edge = InvocationArtifactInputEdgePlan( + key=InvocationArtifactInputProjectionKey(edge.key.invocation_key, 1), + spec=source_spec, + storage_plan=None, + projection=None, + ) + contract = CallableContract.from_callable(consume_labels) + contract = replace( + contract, + module_name="SecondaryConsumer", + metadata=replace(contract.metadata, artifact_inputs=(edge.spec, source_spec)), + ) + adapter = cellprofiler_runtime_adapter_for_test( + runtime_value_store=store, + callable_contract=contract, + artifact_inputs={edge.key: edge, source_edge.key: source_edge}, + source_binding_plan=runtime_input.source_binding_plan, + axis_scope=runtime_input.axis_scope, + ) + + assert adapter.artifact_input_records("Nuclei", ObjectLabelsArtifactType) == (record,) + request = RuntimeInputBindingRequest( + adapter=adapter, kwargs={}, current_image=np.zeros((2, 2)) + ) + assert request.artifact_request(edge).value is record.value.data + + +@pytest.mark.parametrize("component", [AllComponents.SITE, AllComponents.Z_INDEX, AllComponents.TIMEPOINT]) +def test_paired_channel_input_rejects_other_context_coordinate(component): + store, _record, runtime_input = _paired_channel_label_input( + producer_values=((component, "2"),), + ) + + with pytest.raises(RuntimeError, match="Missing RuntimeValueStore record"): + runtime_input.records(store) + + +@pytest.mark.parametrize("producer_axis,producer_path", [ + ("B01", None), + ("A01", "/memory/other_producer/Nuclei.pkl"), +]) +def test_paired_channel_input_rejects_other_well_or_producer(producer_axis, producer_path): + store, _record, runtime_input = _paired_channel_label_input( + producer_axis=producer_axis, producer_path=producer_path, + ) + + with pytest.raises(RuntimeError, match="Missing RuntimeValueStore record"): + runtime_input.records(store) + + +def test_cross_channel_input_without_source_declaration_remains_exact(): + store, _record, runtime_input = _paired_channel_label_input() + strict_input = _ungrouped_runtime_artifact_input( + runtime_input.edge_plan.storage_plan, + axis_scope=runtime_input.axis_scope, + ) + + with pytest.raises(RuntimeError, match="Missing RuntimeValueStore record"): + strict_input.records(store) + + +def test_visible_source_binding_without_context_relation_remains_exact(): + store, _record, runtime_input = _paired_channel_label_input() + unrelated_input = replace( + runtime_input, + edge_plan=replace( + runtime_input.edge_plan, + spec=replace(runtime_input.edge_plan.spec, relations=()), + ), + ) + + with pytest.raises(RuntimeError, match="Missing RuntimeValueStore record"): + unrelated_input.records(store) + + +def test_context_relation_to_another_source_does_not_borrow_visible_plane_membership(): + store, _record, runtime_input = _paired_channel_label_input() + unrelated_input = replace( + runtime_input, + edge_plan=replace( + runtime_input.edge_plan, + spec=replace( + runtime_input.edge_plan.spec, + relations=(InputGroupLineageSourceRelation( + ArtifactSpec.input("UnrelatedImage", ImageArtifactType).ref() + ),), + ), + ), + ) + + with pytest.raises(RuntimeError, match="Missing RuntimeValueStore record"): + unrelated_input.records(store) + + +def test_paired_channel_projection_rejects_a_different_producer_site_plane(): + store, record, runtime_input = _paired_channel_label_input() + edge = runtime_input.edge_plan + payload = replace( + record.value.data, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/source/site2_DNA.tif",), + component_metadata=({"site": "2", "channel": "1"},), + ), + ) + store.replace( + replace(record.value, data=payload), path=record.path, backend=record.backend + ) + projected_input = replace( + runtime_input, + edge_plan=replace( + edge, + storage_plan=replace(edge.storage_plan, variable_components=(AllComponents.SITE,)), + projection=replace( + edge.projection, + component_scopes=(ComponentGroupScope(("1",), component=AllComponents.SITE),), + ), + ), + ) + + with pytest.raises(RuntimeError, match="no producer plane"): + projected_input.resolve_value(store) + + +def test_paired_channel_input_rejects_ambiguous_address_matched_contexts(): + store, _record, runtime_input = _paired_channel_label_input() + _other_store, other_record, _other_input = _paired_channel_label_input( + producer_values=((AllComponents.CHANNEL, "3"),), + ) + store.record(other_record.value, path=other_record.path, backend=other_record.backend) + + with pytest.raises(RuntimeError, match="Ambiguous RuntimeValueStore records"): + runtime_input.records(store) def test_runtime_artifact_input_preserves_same_scope_semantic_partitions(): From ae29c26f39fcceafe0ea5054b57afa4177c232fb Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 15:22:35 -0400 Subject: [PATCH 05/17] Record current-main source build and strengthen missing-source regression --- ..._measurement_contract_checkpoint_20260929.md | 17 +++++++++++++++++ tests/unit/test_runtime_value_store.py | 5 ++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index de30be99be..2d9c0da42d 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -196,3 +196,20 @@ Removed its 608 KiB after preserving evidence and the fully self-seeding source reproducer. Generated arrays and logs are regeneratable; no source or saved sessions were stored there. No owned large scratch artifacts were created. Viewer-bind and ROI materialization implementation surfaces remain with their assigned owners. + +Current-main local build prerequisite: `a0263e82a1` adds the required +`_granularity_reconstruct` extension, reached by intensity -> shape -> zernike -> +granularity imports. A fresh source import failed with `ModuleNotFoundError` +before a test could execute. Zeno owns the bounded worktree-only build using the +existing `setup.py build_ext --inplace` declaration, with native intermediates +under `/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929/native`. +This does not install packages or modify the shared environment or installed +source. The existing build completed in 2.33 seconds with 97,264 KiB maximum +resident memory. Intensity, primary, secondary, and the generated native extension +all import from this worktree; available RAM before the build was 10.8 GiB. +Removed the 136 KiB owned disposable native intermediates after this successful +import check. They are regeneratable. The generated ignored `.abi3.so` stays in +this worktree for source validation. This is import/build evidence, not a complete +headless or installed journey. The strengthened missing-source regression retains +the context relation and removes only its source declaration: one focused case +passes, with 47 unrelated cases deselected in that one-test check. diff --git a/tests/unit/test_runtime_value_store.py b/tests/unit/test_runtime_value_store.py index a4dec14e49..52bf35e264 100644 --- a/tests/unit/test_runtime_value_store.py +++ b/tests/unit/test_runtime_value_store.py @@ -1505,9 +1505,8 @@ def test_paired_channel_input_rejects_other_well_or_producer(producer_axis, prod def test_cross_channel_input_without_source_declaration_remains_exact(): store, _record, runtime_input = _paired_channel_label_input() - strict_input = _ungrouped_runtime_artifact_input( - runtime_input.edge_plan.storage_plan, - axis_scope=runtime_input.axis_scope, + strict_input = replace( + runtime_input, source_binding_plan=CompiledSourceBindingPlan.empty(), ) with pytest.raises(RuntimeError, match="Missing RuntimeValueStore record"): From d7515e76703efff4fe05d22b621a3e7e6b43d532 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 15:32:23 -0400 Subject: [PATCH 06/17] Preserve exact artifact admission for unscoped and partial contexts --- ...easurement_contract_checkpoint_20260929.md | 34 ++++++ .../issue204_catalog_diagnosis_20260929.md | 9 ++ openhcs/core/component_group_scope.py | 8 ++ openhcs/core/runtime_stores.py | 16 ++- tests/unit/test_runtime_value_store.py | 103 ++++++++++++++++-- 5 files changed, 158 insertions(+), 12 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 2d9c0da42d..4cc6353325 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -94,6 +94,35 @@ Runtime checkpoint after the explicit scope extension: - The resource guard reported 11.6 GiB available RAM; its warning was historical 11.5 GiB used swap. Only lightweight source/unit checks proceeded. +Coordinator scope-preservation witness, followed through: + +- Ordinary exact unscoped inputs and producer-only, consumer-only, shared-partial, + and disjoint-partial fixed scopes were explicitly tested through + `RuntimeArtifactInput.records`, not only through a predicate. They already + passed because the earlier projection retained the exact axis key. +- A genuinely empty projected-context witness uses a context-carrying object-label + artifact and an existing source declaration whose plane membership projects + the axis key. The published candidate then reproduced **four failures**: + unscoped, producer-only, consumer-only, and disjoint-partial inputs lost their + exact address-matched record because `bool(shared_keys)` was false. This proves + a focused runtime failure for the witness, not its prevalence in real pipelines. +- Artifact admission now derives shared additional constraints from the typed + scope owners, compiled variable-axis projection, and declared image-set policy. + No shared additional constraint is vacuously satisfied after producer/address/ + group selection. Otherwise the existing typed predicate checks their values. + The artificial shared axis key was removed from this additional-context test. +- `SourceImageSetIdentityCompatibility` is **unchanged**. Each projected-context + case asserts it still rejects identities with no shared evidence; exact artifact + admission has a different, already-proven identity boundary. Wrong well and + producer paths still fail before this additional-constraint check. +- `pytest -q tests/unit/test_runtime_value_store.py --tb=short`: **56 passed**, + 1.67 seconds in the final run including the exact-address negative assertions. + Original conflicting spatial/time, + exact producer/group, retained-plane, and ambiguity assertions still pass. +- Only `core/runtime_stores.py`, its typed scope owner, and their tests were + modified for this witness. No source-binding owner, shared artifact declaration, + CellProfiler adapter, or #214 diagnostic-output field was modified in this step. + ## Essential anti-pattern audit: actual ownership evidence This is a focused source/caller and AST audit against the current archived @@ -110,6 +139,7 @@ That guard detects listed shapes, not every possible semantic ownership defect. | IDEN-7, IMPL-10 | An initial placement incorrectly rejected heterogeneous CellProfiler records. Validation now runs after invocation-contract finalization, where the existing `runtime_adapter.manages_artifact_outputs` capability names the output owner. Native table-wide subject invariants apply only to native returned payloads, not adapter-recorded tables. Real primary/secondary compilation and execution plus native missing/corrected tests exercise both owners. | | AGENT-2/3/6/7, TIME-5/9 | No second mechanism, format converter, test-only production switch, or dormant feature was added. One pre-existing cardinality fixture received a valid artifact-level subject so its original missing-return assertion remains meaningful. The paired-channel failure is retained, not hidden by weaker expectations; publication proceeds with that exact acceptance gap stated. | | IDEN-1/6/7, BOUND-2, MEMB-1/2/3, IMPL-5/12, TIME-1/3 | Exact compiled producer/address selection and contextual source compatibility answer different questions. `RuntimeArtifactInput` derives context membership from `edge.spec.source_context_sources()` and the referenced original source bindings, then delegates matching to the existing typed identity predicate. `RuntimeExecutionAxisScope` owns its typed-coordinate projection. No literal channel/site/z/time matching roster, new registry, global ignore flag, alternate reader, or copied predicate was added. All three constructors are migrated; two actual adapter consumers and fail-closed new cases execute. The replaced fixed-coordinate loops are gone. | +| IDEN-7, BOUND-2, IMPL-5 | Artifact admission asks whether an already-selected producer violates any shared additional coordinate constraint, not whether unknown plane identities prove correspondence. The four failing empty/asymmetric witnesses distinguish these questions. `ComponentSet.intersection`, scope-owned `source_components`, and policy-owned `identity_components()` derive the applicable constraint domain. No copied compatibility matcher or weakened global empty-identity semantics was introduced. | Caller review covered the default invocation declaration provider (returns the finalized `CallableContract`), every call to the new output-validation hook and @@ -154,6 +184,10 @@ duplicate-resolution removal is factoring; no pure file move is claimed. present and store regressions pass, but complete paired execution remains unverified. No `core/source_bindings.py`, `core/source_binding_workspace.py`, source preparation, ROI, viewer, or #211 authorization method/test was modified. +- Scheduling: the parent owns the first released validation slot for real MCP + #151/#212 and its useful merge/installation checkpoint. Zeno will run the full + six-case headless journey only after the explicit next-slot handoff. No race for + fresh environments, GUI, JVM, or installed-baseline changes is authorized. - Source-only explanation of the apparent grouped/ungrouped distinction: group coordinates are selected by the compiled edge, while the previous predicate compared fixed coordinates on either side. Binding-owned channel identities diff --git a/docs/plans/issue204_catalog_diagnosis_20260929.md b/docs/plans/issue204_catalog_diagnosis_20260929.md index d69ee0484d..f7e3da2f8b 100644 --- a/docs/plans/issue204_catalog_diagnosis_20260929.md +++ b/docs/plans/issue204_catalog_diagnosis_20260929.md @@ -42,6 +42,15 @@ The full paired headless journey could not acquire Euler's validation lock (exit 75, no pytest started) and remains pending without deselection. Actual GUI interaction and installed-stack verification also remain coordinator-owned. +The coordinator's empty/asymmetric scope witness is now covered explicitly. +It reproduced four address-matched input failures when declaration projection +left no shared context keys. The artifact boundary now admits vacuous additional +constraints after exact producer selection; the unrelated plane-identity +compatibility predicate remains unchanged and still rejects empty identities. +All 56 runtime-store cases pass after this correction. Full paired execution +awaits the explicit worker validation-slot handoff after the parent's #151/#212 +MCP/installation checkpoint; no first-release race or baseline change will occur. + ## Original observed failure and acceptance (preserved) Original title: Allow typed exact object-label selection for scalar CellProfiler measurement steps diff --git a/openhcs/core/component_group_scope.py b/openhcs/core/component_group_scope.py index 999aef0bd7..78b18d51af 100644 --- a/openhcs/core/component_group_scope.py +++ b/openhcs/core/component_group_scope.py @@ -552,6 +552,14 @@ def source_image_set_identity( ) ) + @property + def source_components(self) -> ComponentSet: + """Return the typed components whose coordinates this scope declares.""" + + return ComponentSet.collect( + (component for component, _value in self.source_component_values) + ) + def matching_component_plane_indices( self, component_metadata: Sequence[Mapping[str, object] | None], diff --git a/openhcs/core/runtime_stores.py b/openhcs/core/runtime_stores.py index b22fec0459..fb4362411a 100644 --- a/openhcs/core/runtime_stores.py +++ b/openhcs/core/runtime_stores.py @@ -572,11 +572,6 @@ def _matches_execution_scope( context_components = ComponentSet.collect( (component for component, _value in self.axis_scope.fixed_component_values), (component for component, _value in record_scope.fixed_component_values), - ( - component - for component in AllComponents - if component.is_multiprocessing_axis() - ), ).excluding(projected_components) # Only a declared source-context relation supplies plane membership. # Unrelated visible bindings must not relax an exact artifact input. @@ -588,6 +583,17 @@ def _matches_execution_scope( if context_sources else SourceImageSetIdentityPolicy() ) + context_components = ( + context_components + .intersection(record_scope.source_components) + .intersection(self.axis_scope.source_components) + .intersection(ComponentSet(identity_policy.identity_components())) + ) + # Exact producer/address/group selection has already proved identity. + # No shared additional constraint is vacuously satisfied here; this does + # not relax the evidence requirement of plane-identity compatibility. + if not context_components: + return True return SourceImageSetIdentityCompatibility( record_scope.source_image_set_identity( identity_policy, components=context_components, diff --git a/tests/unit/test_runtime_value_store.py b/tests/unit/test_runtime_value_store.py index 52bf35e264..50e6dbd111 100644 --- a/tests/unit/test_runtime_value_store.py +++ b/tests/unit/test_runtime_value_store.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from openhcs.constants.constants import AllComponents +from openhcs.constants.constants import AllComponents, get_multiprocessing_axis from openhcs.core.artifacts import ( ArtifactInputProjectionPlan, ArtifactInputPlan, @@ -32,6 +32,7 @@ ) from openhcs.core.component_group_scope import ComponentGroupScope from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.component_set import ComponentSet from openhcs.core.callable_contract import CallableContract from openhcs.core.runtime_equivalence import ( RuntimeMeasurementObservationAxis, @@ -65,6 +66,7 @@ ComponentSelector, NamedSourceBinding, ) +from openhcs.core.source_matching import SourceImageSetIdentityCompatibility, SourceImageSetIdentityPolicy from openhcs.interop.cellprofiler.runtime.artifact_binding import RuntimeInputBindingRequest from tests.unit.cellprofiler_runtime_test_support import cellprofiler_runtime_adapter_for_test @@ -1594,7 +1596,24 @@ def test_runtime_artifact_input_preserves_same_scope_semantic_partitions(): ) -def test_runtime_artifact_input_accepts_consumer_coordinate_absent_from_producer(): +@pytest.mark.parametrize("producer_values,consumer_values", [ + pytest.param((), (), id="exact-unscoped"), + pytest.param((), ((AllComponents.Z_INDEX, "1"),), id="consumer-only"), + pytest.param(((AllComponents.TIMEPOINT, "1"),), (), id="producer-only"), + pytest.param( + ((AllComponents.TIMEPOINT, "1"),), + ((AllComponents.Z_INDEX, "1"),), + id="disjoint-partial", + ), + pytest.param( + ((AllComponents.TIMEPOINT, "1"),), + ((AllComponents.Z_INDEX, "1"), (AllComponents.TIMEPOINT, "1")), + id="shared-partial", + ), +]) +def test_runtime_artifact_input_accepts_exact_unscoped_and_partial_coordinates( + producer_values, consumer_values, +): store = RuntimeValueStore() storage_plan = ArtifactInputPlan( name="positions", @@ -1620,7 +1639,7 @@ def test_runtime_artifact_input_accepts_consumer_coordinate_absent_from_producer "A01", component=None, value=None, - fixed_component_values=((AllComponents.TIMEPOINT, "1"),), + fixed_component_values=producer_values, ), ) record = store.replace(value, path=storage_plan.path, backend="memory") @@ -1628,10 +1647,7 @@ def test_runtime_artifact_input_accepts_consumer_coordinate_absent_from_producer "A01", component=None, value=None, - fixed_component_values=( - (AllComponents.Z_INDEX, "1"), - (AllComponents.TIMEPOINT, "1"), - ), + fixed_component_values=consumer_values, ) assert _ungrouped_runtime_artifact_input( @@ -1640,6 +1656,79 @@ def test_runtime_artifact_input_accepts_consumer_coordinate_absent_from_producer ).records(store) == (record,) +@pytest.mark.parametrize("producer_values,consumer_values", [ + pytest.param((), (), id="empty-projected-context"), + pytest.param(((AllComponents.TIMEPOINT, "1"),), (), id="producer-only"), + pytest.param((), ((AllComponents.Z_INDEX, "1"),), id="consumer-only"), + pytest.param( + ((AllComponents.TIMEPOINT, "1"),), + ((AllComponents.Z_INDEX, "1"),), + id="disjoint-partial", + ), +]) +def test_exact_input_admits_no_shared_projected_context_constraints( + producer_values, consumer_values, +): + _store, original, original_input = _paired_channel_label_input() + storage_plan = original_input.edge_plan.storage_plan + producer_scope = RuntimeExecutionAxisScope.from_raw( + "A01", component=None, value=None, fixed_component_values=producer_values, + ) + store = RuntimeValueStore() + record = store.record( + replace(original.value, key=replace(original.key, scope=producer_scope)), + path=original.path, backend=original.backend, + ) + source = NamedSourceBinding( + alias="Reference", + component_identity=(ComponentSelector(get_multiprocessing_axis(), "A01"),), + ) + consumer_scope = RuntimeExecutionAxisScope.from_raw( + "A01", component=None, value=None, fixed_component_values=consumer_values, + ) + runtime_input = _ungrouped_runtime_artifact_input( + storage_plan, axis_scope=consumer_scope, + source_binding_plan=CompiledSourceBindingPlan(bindings=(source,)), + ) + runtime_input = replace( + runtime_input, + edge_plan=replace( + runtime_input.edge_plan, + spec=runtime_input.edge_plan.spec.with_group_scope_relation( + InputGroupLineageSourceRelation(source.input_spec().ref()) + ), + ), + ) + + # Plane-identity compatibility still requires evidence of a shared identity. + # Exact artifact admission instead has a proven producer/address and only + # checks the additional coordinates constrained on both sides. + assert runtime_input.edge_plan.spec.source_context_sources() == (source.input_spec().ref(),) + policy = SourceImageSetIdentityPolicy.from_source_bindings(runtime_input.source_binding_plan) + components = ComponentSet.collect( + (component for component, _value in producer_scope.source_component_values), + (component for component, _value in consumer_scope.source_component_values), + ) + assert not SourceImageSetIdentityCompatibility( + producer_scope.source_image_set_identity(policy, components=components), + consumer_scope.source_image_set_identity(policy, components=components), + ).matches() + assert runtime_input.records(store) == (record,) + wrong_well = replace( + runtime_input, axis_scope=replace(consumer_scope, axis_id="B01"), + ) + wrong_producer = replace( + runtime_input, + edge_plan=replace( + runtime_input.edge_plan, + storage_plan=replace(storage_plan, path="/memory/other/Nuclei.pkl"), + ), + ) + for rejected_input in (wrong_well, wrong_producer): + with pytest.raises(RuntimeError, match="Missing RuntimeValueStore record"): + rejected_input.records(store) + + def test_runtime_artifact_input_rejects_conflicting_declared_fixed_coordinate(): store = RuntimeValueStore() storage_plan = ArtifactInputPlan( From dd02a00f6cb7fa969abd8a70b0ab9aa2a10b87d7 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 15:43:54 -0400 Subject: [PATCH 07/17] Trace declared paired input context in the continuous selector journey --- ...3_204_measurement_contract_checkpoint_20260929.md | 8 ++++++++ .../test_measurement_declaration_journey.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 4cc6353325..298f52725f 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -188,6 +188,14 @@ duplicate-resolution removal is factoring; no pure file move is claimed. #151/#212 and its useful merge/installation checkpoint. Zeno will run the full six-case headless journey only after the explicit next-slot handoff. No race for fresh environments, GUI, JVM, or installed-baseline changes is authorized. +- Parent also owns the cold-inspection deletion in + `InProcessCompileInspectionGateway`; Zeno does not edit that method or its + catalog-initialization call. The pending continuous regression is strengthened + to verify the secondary declaration's exact source-context relation and the + primary/secondary records' actual typed channel coordinates, in addition to + the existing exact selector and measured-secondary-region assertions. These + additional assertions are syntax-checked only until the worker slot is handed + off; no end-to-end success is inferred from source review. - Source-only explanation of the apparent grouped/ungrouped distinction: group coordinates are selected by the compiled edge, while the previous predicate compared fixed coordinates on either side. Binding-owned channel identities diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index 4aa56fc199..3106b44c05 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -250,6 +250,14 @@ def test_exact_secondary_selector_survives_authoring_compile_and_execution(tmp_p global_config = GlobalPipelineConfig(num_workers=1, use_threading=True) bundle = _compile(tmp_path, reconstructed, global_config) context = bundle.runtime_contexts["A01"] + secondary_invocation = next( + context.step_plans[1].compiled_function_pattern.iter_invocations() + ) + label_input, image_input = IdentifySecondaryObjectsModule.segmentation_inputs( + secondary_invocation.contract.artifact_inputs + ) + assert label_input.source_context_sources() == (image_input.ref(),) + assert image_input.name == ("DNA" if same_source else "Actin") plan = context.step_plans[2] invocation = next(plan.compiled_function_pattern.iter_invocations()) (edge,) = ( @@ -267,6 +275,10 @@ def test_exact_secondary_selector_survives_authoring_compile_and_execution(tmp_p store = context.runtime_value_store [primary] = store.find(name="Nuclei", axis_id="A01") [secondary] = store.find(name="Cells", axis_id="A01") + assert primary.key.scope.value_text_for_component(AllComponents.CHANNEL) == "1" + assert secondary.key.scope.value_text_for_component(AllComponents.CHANNEL) == ( + "1" if same_source else "2" + ) primary_area = np.count_nonzero(object_label_dense_array(primary.value.data)) secondary_area = np.count_nonzero(object_label_dense_array(secondary.value.data)) assert secondary_area > primary_area > 0 From a38be98d38d1bb7a6b5cff135e7e1e4903818207 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 16:21:14 -0400 Subject: [PATCH 08/17] Validate all output kinds through declared recording policies --- ...easurement_contract_checkpoint_20260929.md | 89 +++++- openhcs/core/artifact_key_selection.py | 60 +++- openhcs/core/artifacts.py | 33 ++- openhcs/core/callable_contract.py | 17 +- openhcs/core/function_patterns.py | 3 +- openhcs/core/runtime_adapters.py | 19 +- .../interop/cellprofiler/runtime/adapter.py | 20 +- .../cellprofiler/runtime/module_execution.py | 2 +- .../runtime/output_record_request.py | 4 +- .../cellprofiler/runtime/output_recording.py | 7 +- tests/unit/test_artifact_output_ownership.py | 260 ++++++++++++++++++ tests/unit/test_function_artifact_outputs.py | 3 +- tests/unit/test_function_patterns.py | 35 ++- ...test_function_runtime_source_projection.py | 7 +- .../unit/test_function_step_execution_plan.py | 7 +- .../unit/test_path_planner_materialization.py | 14 +- 16 files changed, 514 insertions(+), 66 deletions(-) create mode 100644 tests/unit/test_artifact_output_ownership.py diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 298f52725f..8e455bc999 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -2,7 +2,7 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. -Base: `openhcsdev/main` at `a0263e82a1`, fetched and merged normally on 2026-09-29. +Base: `openhcsdev/main` at `283b21275`, fetched and merged normally on 2026-09-29. The earlier checkpoint used `98d9b9d23`; the runtime checkpoint integrates new main through normal merge `9f5d444515`. No parent/main-worktree edits, rebases, resets, installed-source edits, or gitlink changes. @@ -17,10 +17,13 @@ open. This is a source-verified draft, not an installed/live-readiness claim. CSV materialization do not imply a measurement subject. - Compile and runtime resolve subjects through the original relation owners. The error identifies the output and explains image, object, and artifact relations. -- The finalized callable contract delegates adapter-recorded outputs to the - existing runtime adapter owner (`manages_artifact_outputs`). Native returns use - the artifact-kind requirement. This distinction is required: CellProfiler - primary/secondary modules record heterogeneous measurements themselves. +- The finalized callable contract projects its declared `artifact_output_policy`. + Every policy runs the artifact kind's common declaration invariants, then its + native-return or adapter-recorded payload obligation. The old blanket exemption + and `manages_artifact_outputs` field are deleted, with all callers cut over. + Input management remains an independent capability. CellProfiler's policy + requires the existing module measurement owner rather than a table-wide subject; + its original row policy and recording boundary still validate actual subjects. - CellProfiler catalog bindings project `require_parameter_name()`, not the optional raw `parameter_name`. The existing `select_object_sets_to_measure` selector is discoverable separately from the runtime-owned scalar `labels` parameter. @@ -44,6 +47,48 @@ eight recorded submodule `src` directories on `PYTHONPATH`; all nine imported package paths were verified inside this worktree. Submodules were initialized to their recorded gitlinks without installed-source changes. +### Architectural review follow-through + +Review: https://github.com/OpenHCSDev/openhcs/pull/205#issuecomment-5897282604. +The review correctly identified a consumer-owned exemption in the previous +checkpoint. Its earlier audit claim below was too strong: skipping every kind +hook was not an adequate expression of CP's heterogeneous table obligation. + +- Red witness at the pre-fix body: the new materialization-bearing output kind + produced **4 failed, 4 passed**, 1.65 seconds. All four adapter-recorded cases + silently bypassed its invariant; native-return cases rejected the declaration. + Contract and compile boundaries, with input management both on and off, expose + the same defect. Corrected declarations also compile without executing a callable. +- `tests/unit/test_artifact_output_ownership.py` now exercises the same new kind + under native, generic adapter-recorded and CP-recorded policies without any + compiler/consumer edit for that kind. It also checks single-subject requirements, + common conflicting-subject rejection, CP's required module row owner, and the + nominal policy boundary. The real `CellProfilerInvocationContractProviderFactory` + derives a primary module contract with its original measurement owner and no + table-wide subject. Real `compile_function_pattern` admits that contract; + replacing only its recording owner with native returns rejects it. +- Final output-policy file: **20 passed**. The preceding five-file run covering + output policy, native artifact outputs, all runtime-store regressions, catalog + friction and artifact identity reconstruction: **153 passed**, 4.95 seconds + (17 policy cases before the final three nominal-boundary cases were added). +- `test_function_patterns.py` plus `test_function_step_execution_plan.py`: + **80 passed** in the 95-case run including the then-15 policy cases. Four generic + adapter measurement fixtures previously omitted subjects; they now declare valid + artifact subjects while retaining their exact component/projection assertions. +- Three selected planner cases: **3 passed, 96 deselected**, 1.92 seconds. + Exact managed group lineage and main-flow preservation retain their assertions. +- Existing `test_track_objects_record_builder_uses_nominal_image_table_ownership`: + **1 passed, 393 deselected**, 5.08 seconds. The actual module row builder emits + mixed image/object tracking rows with their declared per-row owners. This is + provider-free record-building evidence, not complete pipeline/live execution. +- Existing adapter output runtime-context regression: + **1 passed, 91 deselected**, 1.71 seconds. The separate adapter component-output + chain regression also passes; no MCP/JVM/GUI process or environment was started. +- All checks use the required interpreter, worktree and recorded submodule paths. + Latest guard: **11.5 GiB available RAM**, historical swap warning at 11.9 GiB. + These are bounded source/unit checks, not a second heavy validation slot. No + installed source, shared environment, skill, or frozen runtime was modified. + - Initial published checkpoint: 12 catalog/identity tests and 6 focused native subject tests passed. - Full focused rerun: `pytest -q tests/unit/test_function_artifact_outputs.py @@ -127,16 +172,16 @@ Coordinator scope-preservation witness, followed through: This is a focused source/caller and AST audit against the current archived refactor-audit pattern catalog, **not a complete NRA semantic/proof scan**. All -eleven changed production files were parsed against current merged main: zero added +eleven changed production files in the earlier runtime checkpoint were parsed against its merged main: zero added `match`, `isinstance`, `getattr`/`hasattr`, or 6+-term boolean-chain AST nodes. That guard detects listed shapes, not every possible semantic ownership defect. | Pattern IDs | Owner, projection, and new-case evidence | | --- | --- | -| IMPL-1/2/3/4, MEMB-1/2 | The finalized compiler calls `ArtifactPlanKeySelector.validate_artifact_output_declarations`; each spec delegates to `spec.artifact_type.validate_output_declaration`. Only `MeasurementsArtifactType` owns the subject requirement. No kind/name roster or external type switch was added. A future output kind supplies its rule on its declaration; compile/runtime consumers need no new branch. | +| IMPL-1/2/3/4, MEMB-1/2 | The finalized compiler calls the selector's public validation contract. `ArtifactOutputPolicy` runs every kind's common hook, then the policy dispatches to the kind's native-return or adapter-recorded obligation. `MeasurementsArtifactType` owns common relation coherence and the native subject requirement; CP declares its distinct module-row-owner obligation. The new-kind materialization rule requires one declaration and zero compiler/consumer changes, and executes under all three owners. No kind/name roster or concrete-adapter switch was added. | | IMPL-5/12/13, TIME-1/3 | `ArtifactSpecRelation.measurement_subject_for_output` resolves original relation-owned subjects once. The output-plan accessor, kind requirement, and native runtime all delegate; the former output-plan extraction body and runtime missing-subject body were deleted in place. The output-plan method remains a domain accessor, not an old reader or compatibility alias. | | IDEN-1/2/5, BOUND-2/7, MEMB-3/4 | `SettingToKeywordBinding.require_parameter_name()` owns selector identity, separately from `runtime_parameter_name`. The only production `CellProfilerArtifactBindingSummary` constructor projects that owner; its selector field is now required. No selector store, guessed names, raw-row decoder, or attribute-name probe exists in the change. The catalog regression derives every registered binding and exercises new derived-name input and explicit-name output declarations without editing the projection. | -| IDEN-7, IMPL-10 | An initial placement incorrectly rejected heterogeneous CellProfiler records. Validation now runs after invocation-contract finalization, where the existing `runtime_adapter.manages_artifact_outputs` capability names the output owner. Native table-wide subject invariants apply only to native returned payloads, not adapter-recorded tables. Real primary/secondary compilation and execution plus native missing/corrected tests exercise both owners. | +| IDEN-1/7, IMPL-4/10, IMPL-5/12 | Recording ownership no longer answers whether common compile invariants may run. The blanket early return and output flag were removed, not relocated; native/adapter policies have distinct kind-dispatched payload obligations. CP requires its already-declared `measurement_feature_owner`, leaving actual heterogeneous row validation with the original module row policy and `add_measurements`. Common conflicting-subject checks stay on the kind. The new-case matrix varies input management independently, and the real CP provider/admission plus mixed tracking-row builder pass. | | AGENT-2/3/6/7, TIME-5/9 | No second mechanism, format converter, test-only production switch, or dormant feature was added. One pre-existing cardinality fixture received a valid artifact-level subject so its original missing-return assertion remains meaningful. The paired-channel failure is retained, not hidden by weaker expectations; publication proceeds with that exact acceptance gap stated. | | IDEN-1/6/7, BOUND-2, MEMB-1/2/3, IMPL-5/12, TIME-1/3 | Exact compiled producer/address selection and contextual source compatibility answer different questions. `RuntimeArtifactInput` derives context membership from `edge.spec.source_context_sources()` and the referenced original source bindings, then delegates matching to the existing typed identity predicate. `RuntimeExecutionAxisScope` owns its typed-coordinate projection. No literal channel/site/z/time matching roster, new registry, global ignore flag, alternate reader, or copied predicate was added. All three constructors are migrated; two actual adapter consumers and fail-closed new cases execute. The replaced fixed-coordinate loops are gone. | | IDEN-7, BOUND-2, IMPL-5 | Artifact admission asks whether an already-selected producer violates any shared additional coordinate constraint, not whether unknown plane identities prove correspondence. The four failing empty/asymmetric witnesses distinguish these questions. `ComponentSet.intersection`, scope-owned `source_components`, and policy-owned `identity_components()` derive the applicable constraint domain. No copied compatibility matcher or weakened global empty-identity semantics was introduced. | @@ -149,6 +194,15 @@ still checks relation references without prematurely applying native runtime requirements to partially reconstructed CellProfiler contracts. Real compile inspection reaches the finalized validation boundary and fails before execution. +Review follow-through at `dd02a00f6` plus the latest normal main merge covered all +output-policy constructors, decorator calls, compiler hooks and four production +recording consumers. `rg` finds no remaining `manages_artifact_outputs` use in +production or tests; there is no compatibility alias. The nine production files +changed for this review were AST parsed: zero added `match`, attribute-name probes +or 6+-term boolean chains; one added `isinstance` is the nominal output-policy +declaration boundary, tested for invalid/abstract policies. This is focused +source/caller/new-case coverage, not complete NRA detector or native-proof coverage. + Runtime caller review also covered `records`, static/dynamic `all_records`, `_records`, `_validate_selected_records`, `_axis_value`, `projected_values`, `composed_value`, all three production `RuntimeArtifactInput` constructors, and @@ -179,15 +233,15 @@ duplicate-resolution removal is factoring; no pure file move is claimed. secondary source's channel 2 scope and rejects it. Address, axis, and producer identity match. Reproduce with `pytest -q tests/integration/test_measurement_declaration_journey.py -k paired --tb=short` - under the serialized validation lock when Euler releases it. Zeno is now the + under the serialized validation lock after coordinator handoff. Zeno is the active runtime-fix owner under the explicit scope extension. The focused fix is present and store regressions pass, but complete paired execution remains unverified. No `core/source_bindings.py`, `core/source_binding_workspace.py`, source preparation, ROI, viewer, or #211 authorization method/test was modified. -- Scheduling: the parent owns the first released validation slot for real MCP - #151/#212 and its useful merge/installation checkpoint. Zeno will run the full - six-case headless journey only after the explicit next-slot handoff. No race for - fresh environments, GUI, JVM, or installed-baseline changes is authorized. +- Scheduling: parent #151/#212 acceptance and installation are complete. + Confucius now owns the heavy runtime lock and frozen installation. Zeno will run + the full six-case headless journey only after an explicit serialized handoff. + No race for fresh environments, GUI, JVM, or installed-baseline changes is authorized. - Parent also owns the cold-inspection deletion in `InProcessCompileInspectionGateway`; Zeno does not edit that method or its catalog-initialization call. The pending continuous regression is strengthened @@ -220,11 +274,20 @@ Draft PR: https://github.com/OpenHCSDev/openhcs/pull/205. - `openhcs/core/artifacts.py` - `openhcs/core/callable_contract.py` - `openhcs/core/function_patterns.py` +- `openhcs/core/runtime_adapters.py` - `openhcs/core/component_group_scope.py` - `openhcs/core/runtime_stores.py` - `openhcs/core/steps/function_runtime.py` - `openhcs/interop/cellprofiler/runtime/adapter.py` - `openhcs/interop/cellprofiler/runtime/artifact_binding.py` +- `openhcs/interop/cellprofiler/runtime/module_execution.py` +- `openhcs/interop/cellprofiler/runtime/output_record_request.py` +- `openhcs/interop/cellprofiler/runtime/output_recording.py` +- `tests/unit/test_artifact_output_ownership.py` +- `tests/unit/test_function_patterns.py` +- `tests/unit/test_function_runtime_source_projection.py` +- `tests/unit/test_function_step_execution_plan.py` +- `tests/unit/test_path_planner_materialization.py` - `tests/unit/agent/test_mcp_cellprofiler_function_contract_friction.py` - `tests/unit/test_function_artifact_outputs.py` - `tests/unit/test_runtime_value_store.py` diff --git a/openhcs/core/artifact_key_selection.py b/openhcs/core/artifact_key_selection.py index ab29316ad7..bf0fe59522 100644 --- a/openhcs/core/artifact_key_selection.py +++ b/openhcs/core/artifact_key_selection.py @@ -4,18 +4,65 @@ from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TypeVar +from typing import ClassVar, TypeVar from openhcs.core.artifacts import ( ArtifactOutputPlan, ArtifactPlan, + ArtifactSpec, ArtifactSpecCollection, ArtifactSpecRef, + MeasurementsArtifactType, ) ArtifactPlanT = TypeVar("ArtifactPlanT", bound=ArtifactPlan) +class ArtifactOutputPolicy(ABC): + """Declaration-owned recording and payload obligations for output artifacts.""" + + records_outputs: ClassVar[bool] + + @classmethod + def validate_output_declarations(cls, specs: ArtifactSpecCollection) -> None: + """Run common kind invariants before the ownership-specific obligation.""" + + for spec in specs.for_plan_type(ArtifactOutputPlan): + spec.artifact_type.validate_output_declaration(spec) + cls.validate_payload_declaration(spec) + + @classmethod + @abstractmethod + def validate_payload_declaration(cls, spec: ArtifactSpec) -> None: + """Validate what this owner needs to turn the declaration into a payload.""" + + +class NativeReturnArtifactOutputPolicy(ArtifactOutputPolicy): + """Returned payloads are normalized and recorded by the native runtime.""" + + records_outputs = False + + @classmethod + def validate_payload_declaration(cls, spec: ArtifactSpec) -> None: + spec.artifact_type.validate_native_output_declaration(spec) + + +class AdapterRecordedArtifactOutputPolicy(ArtifactOutputPolicy): + """The declared adapter records payloads without bypassing kind invariants.""" + + records_outputs = True + + @classmethod + def validate_payload_declaration(cls, spec: ArtifactSpec) -> None: + spec.artifact_type.validate_recorded_output_declaration(spec, cls) + + @classmethod + def validate_measurement_subject(cls, spec: ArtifactSpec) -> None: + """Ordinary recorded tables have one declaration-owned subject.""" + + MeasurementsArtifactType.require_output_subject(spec) + + class ArtifactPlanKeySelector(ABC): """Nominal interface for declarations that select compiled artifact plans.""" @@ -55,11 +102,16 @@ def select_plans( selected.append(plan) return tuple(selected) + @property + def artifact_output_policy(self) -> type[ArtifactOutputPolicy]: + """Return the native output owner unless this declaration supplies another.""" + + return NativeReturnArtifactOutputPolicy + def validate_artifact_output_declarations(self) -> None: - """Validate output kinds when this declaration owns returned payloads.""" + """Validate every output kind through its declared payload owner.""" - for spec in self.artifact_specs.for_plan_type(ArtifactOutputPlan).specs: - spec.artifact_type.validate_output_declaration(spec) + self.artifact_output_policy.validate_output_declarations(self.artifact_specs) def validate_artifact_relation_refs(self, *, owner_name: str) -> None: self.artifact_specs.validate_registered_relation_refs( diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index 203f28ebfa..8059b9db98 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -23,6 +23,7 @@ from openhcs.core.component_set import ComponentSet if TYPE_CHECKING: + from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.runtime_artifact_values import ( RuntimeValue, ) @@ -144,10 +145,26 @@ def diagnostic_label(cls) -> str: @classmethod def validate_output_declaration(cls, spec: "ArtifactSpec") -> None: - """Validate kind-owned output requirements before runtime planning.""" + """Validate common kind invariants regardless of payload recording owner.""" del spec + @classmethod + def validate_native_output_declaration(cls, spec: "ArtifactSpec") -> None: + """Validate kind requirements for native returned payloads.""" + + del spec + + @classmethod + def validate_recorded_output_declaration( + cls, + spec: "ArtifactSpec", + policy: "type[AdapterRecordedArtifactOutputPolicy]", + ) -> None: + """Validate kind requirements for declaration-owned adapter recording.""" + + del spec, policy + @classmethod def normalize_group_scoped_payload( cls, @@ -691,8 +708,22 @@ def require_output_subject( @classmethod def validate_output_declaration(cls, spec: "ArtifactSpec") -> None: + """Explicit subject relations must agree under every recording owner.""" + + ArtifactSpecRelation.measurement_subject_for_output(spec) + + @classmethod + def validate_native_output_declaration(cls, spec: "ArtifactSpec") -> None: cls.require_output_subject(spec) + @classmethod + def validate_recorded_output_declaration( + cls, + spec: "ArtifactSpec", + policy: "type[AdapterRecordedArtifactOutputPolicy]", + ) -> None: + policy.validate_measurement_subject(spec) + @classmethod def runtime_parameter_types(cls) -> tuple[type, ...]: from openhcs.core.runtime_tabular_values import ColumnarRows diff --git a/openhcs/core/callable_contract.py b/openhcs/core/callable_contract.py index cc3bfe1a09..8f4fd94492 100644 --- a/openhcs/core/callable_contract.py +++ b/openhcs/core/callable_contract.py @@ -40,7 +40,10 @@ from openhcs.core.image_payload_execution_mode import ( ImagePayloadExecutionMode, ) -from openhcs.core.artifact_key_selection import ArtifactPlanKeySelector +from openhcs.core.artifact_key_selection import ( + ArtifactOutputPolicy, + ArtifactPlanKeySelector, +) from openhcs.core.artifacts import ( ArtifactSpec, ArtifactSpecCollection, @@ -1146,13 +1149,13 @@ def bind_runtime_owned(parameter_name: str) -> None: ) from exc return tuple(kwargs.items()) - def validate_artifact_output_declarations(self) -> None: - """Validate native returns, leaving adapter-recorded rows with their owner.""" - + @property + def artifact_output_policy(self) -> type[ArtifactOutputPolicy]: + """Project the output policy from the callable's adapter declaration.""" adapter = self.runtime_adapter - if adapter is not None and adapter.manages_artifact_outputs: - return - super(CallableContract, self).validate_artifact_output_declarations() + if adapter is None: + return super(CallableContract, self).artifact_output_policy + return adapter.artifact_output_policy def validate_artifact_input_parameter_bindings(self) -> None: """Validate exact artifact occurrences against the normalized callable ABI.""" diff --git a/openhcs/core/function_patterns.py b/openhcs/core/function_patterns.py index d9e1523044..521851fbee 100644 --- a/openhcs/core/function_patterns.py +++ b/openhcs/core/function_patterns.py @@ -550,8 +550,7 @@ def adapter_records_artifact_outputs(self) -> bool: """Return whether this invocation's adapter records selected outputs.""" return bool( self.artifact_output_plans - and self.contract.runtime_adapter is not None - and self.contract.runtime_adapter.manages_artifact_outputs + and self.contract.artifact_output_policy.records_outputs ) @property diff --git a/openhcs/core/runtime_adapters.py b/openhcs/core/runtime_adapters.py index d346a74058..6c46d7c729 100644 --- a/openhcs/core/runtime_adapters.py +++ b/openhcs/core/runtime_adapters.py @@ -10,6 +10,10 @@ from python_introspect import add_parameter_exclusions from openhcs.constants.constants import Backend, VariableComponents +from openhcs.core.artifact_key_selection import ( + ArtifactOutputPolicy, + NativeReturnArtifactOutputPolicy, +) from openhcs.core.aligned_image_payload import ( ImagePayloadExecutionMode, stack_image_payloads, @@ -454,7 +458,7 @@ class RuntimeAdapterSpec: parameter_name: str factory: RuntimeAdapterFactory manages_artifact_inputs: bool = False - manages_artifact_outputs: bool = False + artifact_output_policy: type[ArtifactOutputPolicy] = NativeReturnArtifactOutputPolicy runtime_callable_factory: RuntimeCallableFactory | None = None def __post_init__(self) -> None: @@ -462,6 +466,15 @@ def __post_init__(self) -> None: raise ValueError("RuntimeAdapterSpec.parameter_name cannot be empty.") if not callable(self.factory): raise TypeError("RuntimeAdapterSpec.factory must be callable.") + if ( + not isinstance(self.artifact_output_policy, type) + or not issubclass(self.artifact_output_policy, ArtifactOutputPolicy) + or inspect.isabstract(self.artifact_output_policy) + ): + raise TypeError( + "RuntimeAdapterSpec.artifact_output_policy requires a concrete " + "ArtifactOutputPolicy declaration." + ) if self.runtime_callable_factory is not None and not callable( self.runtime_callable_factory ): @@ -500,7 +513,7 @@ def runtime_adapter( factory: RuntimeAdapterFactory, *, manages_artifact_inputs: bool = False, - manages_artifact_outputs: bool = False, + artifact_output_policy: type[ArtifactOutputPolicy] = NativeReturnArtifactOutputPolicy, runtime_callable_factory: RuntimeCallableFactory | None = None, ) -> Callable[[_F], _F]: """Declare that a callable needs an invocation-scoped runtime adapter.""" @@ -508,7 +521,7 @@ def runtime_adapter( parameter_name=parameter_name, factory=factory, manages_artifact_inputs=manages_artifact_inputs, - manages_artifact_outputs=manages_artifact_outputs, + artifact_output_policy=artifact_output_policy, runtime_callable_factory=runtime_callable_factory, ) diff --git a/openhcs/interop/cellprofiler/runtime/adapter.py b/openhcs/interop/cellprofiler/runtime/adapter.py index 21592a7d79..9ae407b663 100644 --- a/openhcs/interop/cellprofiler/runtime/adapter.py +++ b/openhcs/interop/cellprofiler/runtime/adapter.py @@ -10,8 +10,10 @@ import numpy as np from openhcs.constants.constants import Backend +from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.artifacts import ( ArtifactOutputPlan, + ArtifactSpec, ArtifactType, ImageArtifactType, MeasurementsArtifactType, @@ -69,6 +71,22 @@ RelationshipIdVector = np.ndarray | Sequence[int] +class CellProfilerRecordedArtifactOutputPolicy(AdapterRecordedArtifactOutputPolicy): + """CP recorders supply per-row subjects for heterogeneous measurement tables.""" + + @classmethod + def validate_measurement_subject(cls, spec: ArtifactSpec) -> None: + # Common relation invariants are checked by the artifact kind. CP's + # module owner assembles heterogeneous rows and validates their subjects + # through its row policy and add_measurements at recording time. + if spec.measurement_feature_owner is None: + raise ValueError( + f"CellProfiler measurement output {spec.ref()!r} requires its " + "declared measurement_feature_owner. Finalize the module's " + "callable contract before compiling heterogeneous rows." + ) + + @dataclass(slots=True) class CellProfilerRuntimeAdapter(RuntimePlaneAxisProjector): """CellProfiler-like API backed by typed OpenHCS runtime state. @@ -98,7 +116,7 @@ def runtime_adapter_spec(cls): parameter_name=cls.require_parameter_name(), factory=cellprofiler_runtime_adapter_factory, manages_artifact_inputs=True, - manages_artifact_outputs=True, + artifact_output_policy=CellProfilerRecordedArtifactOutputPolicy, runtime_callable_factory=cellprofiler_runtime_callable_factory, ) diff --git a/openhcs/interop/cellprofiler/runtime/module_execution.py b/openhcs/interop/cellprofiler/runtime/module_execution.py index f076bfb092..840624bfaf 100644 --- a/openhcs/interop/cellprofiler/runtime/module_execution.py +++ b/openhcs/interop/cellprofiler/runtime/module_execution.py @@ -170,7 +170,7 @@ def __post_init__(self) -> None: if ( runtime_adapter is None or not runtime_adapter.manages_artifact_inputs - or not runtime_adapter.manages_artifact_outputs + or not self.callable_contract.artifact_output_policy.records_outputs ): raise TypeError( "CellProfilerModuleExecutor requires a RuntimeAdapterSpec that " diff --git a/openhcs/interop/cellprofiler/runtime/output_record_request.py b/openhcs/interop/cellprofiler/runtime/output_record_request.py index 1803423c5e..559fda7c41 100644 --- a/openhcs/interop/cellprofiler/runtime/output_record_request.py +++ b/openhcs/interop/cellprofiler/runtime/output_record_request.py @@ -105,11 +105,9 @@ def artifact_output_value( ref = spec.ref() output_plan = self.adapter.request.artifact_output_plan(ref) - runtime_adapter = self.callable_contract.runtime_adapter recorded = bool( output_plan is not None - and runtime_adapter is not None - and runtime_adapter.manages_artifact_outputs + and self.callable_contract.artifact_output_policy.records_outputs ) transient = ref in self.declared_only_outputs match recorded, transient: diff --git a/openhcs/interop/cellprofiler/runtime/output_recording.py b/openhcs/interop/cellprofiler/runtime/output_recording.py index 7da0236489..8e95504e27 100644 --- a/openhcs/interop/cellprofiler/runtime/output_recording.py +++ b/openhcs/interop/cellprofiler/runtime/output_recording.py @@ -89,10 +89,9 @@ def transient_output_values( ) -> Mapping[ArtifactSpecRef, RuntimeCallableArgument]: """Return callable outputs not recorded by this active invocation.""" - runtime_adapter = callable_contract.runtime_adapter recorded_refs = ( frozenset(plan.ref() for plan in active_output_plans) - if runtime_adapter is not None and runtime_adapter.manages_artifact_outputs + if callable_contract.artifact_output_policy.records_outputs else frozenset() ) return MappingProxyType( @@ -144,10 +143,8 @@ def record_module_outputs( active_output_plans=active_output_plans, returned_values=returned_values, ) - runtime_adapter = callable_contract.runtime_adapter if ( - runtime_adapter is None - or not runtime_adapter.manages_artifact_outputs + not callable_contract.artifact_output_policy.records_outputs or not active_output_plans ): return declared_only_outputs diff --git a/tests/unit/test_artifact_output_ownership.py b/tests/unit/test_artifact_output_ownership.py new file mode 100644 index 0000000000..efae325793 --- /dev/null +++ b/tests/unit/test_artifact_output_ownership.py @@ -0,0 +1,260 @@ +"""Compile invariants apply independently of artifact recording ownership.""" + +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from openhcs.core.artifacts import ( + ArtifactSpec, + ArtifactType, + ArtifactMeasurementSubjectRelation, + ImageArtifactType, + ImageMeasurementSubjectRelation, + MeasurementsArtifactType, +) +from openhcs.core.artifact_key_selection import ( + AdapterRecordedArtifactOutputPolicy, + ArtifactOutputPolicy, + NativeReturnArtifactOutputPolicy, +) +from openhcs.core.callable_contract import CallableContract +from openhcs.core.function_patterns import ( + compile_function_pattern, + normalize_function_pattern, +) +from openhcs.core.pipeline.function_contracts import artifact_outputs +from openhcs.core.runtime_adapters import RuntimeAdapterSpec, runtime_adapter +from openhcs.interop.cellprofiler.runtime.adapter import ( + CellProfilerRecordedArtifactOutputPolicy, +) +from openhcs.processing.materialization import CsvOptions, MaterializationSpec + + +@pytest.mark.parametrize("invalid_policy", [None, False, ArtifactOutputPolicy]) +def test_adapter_declaration_requires_a_concrete_output_policy(invalid_policy): + with pytest.raises(TypeError, match="concrete ArtifactOutputPolicy declaration"): + RuntimeAdapterSpec( + "runtime", + lambda request: object(), + artifact_output_policy=invalid_policy, + ) + + +@pytest.fixture +def materialized_output_kind(monkeypatch): + # Isolate the declaration, not the compiler or its registration mechanism. + monkeypatch.setattr(ArtifactType, "__registry__", dict(ArtifactType.__registry__)) + + class MaterializedAuditArtifactType(ArtifactType): + value = "materialized_output_policy_audit" + + @classmethod + def validate_output_declaration(cls, spec): + if spec.materialization is None: + raise ValueError("Audit output requires declared materialization") + + return MaterializedAuditArtifactType + + +@pytest.mark.parametrize("boundary", ["contract", "compile"]) +@pytest.mark.parametrize( + "output_policy", + [ + NativeReturnArtifactOutputPolicy, + AdapterRecordedArtifactOutputPolicy, + CellProfilerRecordedArtifactOutputPolicy, + ], + ids=["native", "recorded", "cellprofiler-recorded"], +) +@pytest.mark.parametrize( + "manages_inputs", [False, True], ids=["native-inputs", "adapter-inputs"] +) +def test_new_output_kind_invariant_is_not_exempted_by_recording_owner( + materialized_output_kind, + boundary, + output_policy, + manages_inputs, +): + spec = ArtifactSpec.output("Audit", materialized_output_kind) + + def declared(output): + @runtime_adapter( + "runtime", + lambda request: object(), + manages_artifact_inputs=manages_inputs, + artifact_output_policy=output_policy, + ) + @artifact_outputs(output) + def publish(image, *, runtime): + raise AssertionError("Compile validation must not execute a callable") + + return publish + + def validate(func): + if boundary == "contract": + CallableContract.from_callable(func).validate_artifact_output_declarations() + else: + compile_function_pattern(func, {}, {}) + + with pytest.raises(ValueError, match="requires declared materialization"): + validate(declared(spec)) + validate(declared(replace(spec, materialization=MaterializationSpec(CsvOptions())))) + + +@pytest.mark.parametrize( + "output_policy", + [ + NativeReturnArtifactOutputPolicy, + AdapterRecordedArtifactOutputPolicy, + ], +) +def test_single_subject_output_policy_rejects_missing_measurement_subject( + output_policy, +): + @runtime_adapter( + "runtime", lambda request: object(), artifact_output_policy=output_policy + ) + @artifact_outputs(ArtifactSpec.output("Counts", MeasurementsArtifactType)) + def count(image, *, runtime): + raise AssertionError("Invalid declarations must not execute") + + with pytest.raises(ValueError, match="no declared measurement subject"): + compile_function_pattern(count, {}, {}) + + +def test_cellprofiler_recording_policy_still_rejects_conflicting_explicit_subjects(): + image = ArtifactSpec.output("Image", ImageArtifactType) + measurements = ArtifactSpec.output( + "Measurements", + MeasurementsArtifactType, + relations=( + ImageMeasurementSubjectRelation(image.ref()), + ArtifactMeasurementSubjectRelation(), + ), + ) + + @runtime_adapter( + "runtime", + lambda request: object(), + artifact_output_policy=CellProfilerRecordedArtifactOutputPolicy, + ) + @artifact_outputs(image, measurements) + def count(image, *, runtime): + raise AssertionError("Invalid declarations must not execute") + + with pytest.raises(ValueError, match="multiple measurement subjects"): + compile_function_pattern(count, {}, {}) + + +def test_cellprofiler_recording_requires_its_declared_measurement_row_owner(): + @runtime_adapter( + "runtime", + lambda request: object(), + artifact_output_policy=CellProfilerRecordedArtifactOutputPolicy, + ) + @artifact_outputs(ArtifactSpec.output("Measurements", MeasurementsArtifactType)) + def measure(image, *, runtime): + raise AssertionError("Invalid declarations must not execute") + + with pytest.raises(ValueError, match="declared measurement_feature_owner"): + compile_function_pattern(measure, {}, {}) + + +def test_real_cellprofiler_declaration_compiles_without_a_table_wide_subject(): + """The real module/provider declares the owner, not a compiler exemption.""" + from openhcs.core.compiled_step_plan import CompiledStepPlan + from openhcs.core.config import GlobalPipelineConfig, PipelineConfig + from openhcs.core.context.processing_context import ProcessingContext + from openhcs.core.invocation_artifacts import ArtifactDeclarationStepContext + from openhcs.core.pipeline.compilation_session import CompilationSession + from openhcs.core.pipeline.step_snapshot import StepSnapshot + from openhcs.core.source_bindings import ( + NamedSourceBinding, + StepSourceBindingsConfig, + ) + from openhcs.core.steps.function_step import FunctionStep + from openhcs.interop.cellprofiler.compile_time_contracts import ( + CellProfilerInvocationContractProviderFactory, + ) + from openhcs.processing.backends.cellprofiler.primary_objects import ( + IdentifyPrimaryObjectsModule, + identify_primary_objects, + ) + from openhcs.core.artifacts import ( + ArtifactInputPlan, + ArtifactOutputPlan, + ArtifactSpecRelation, + ObjectLabelsArtifactType, + ) + + (input_binding,) = IdentifyPrimaryObjectsModule.declared_artifact_bindings( + plan_type=ArtifactInputPlan, + artifact_type=ImageArtifactType, + ) + (output_binding,) = IdentifyPrimaryObjectsModule.declared_artifact_bindings( + plan_type=ArtifactOutputPlan, + artifact_type=ObjectLabelsArtifactType, + ) + step = FunctionStep( + func=( + identify_primary_objects, + { + input_binding.require_parameter_name(): "DNA", + output_binding.require_parameter_name(): "Nuclei", + }, + ), + name="Primary", + source_bindings=StepSourceBindingsConfig( + enabled=True, + bindings=(NamedSourceBinding(alias="DNA"),), + ), + ) + session = CompilationSession.from_context( + context=ProcessingContext( + axis_id="A01", + step_plans={ + 0: CompiledStepPlan( + step_index=0, + step_name=step.name, + step_type="FunctionStep", + axis_id="A01", + ), + }, + ), + steps=[step], + orchestrator=SimpleNamespace(pipeline_config=PipelineConfig()), + global_config=GlobalPipelineConfig(), + step_state_map={0: object()}, + snapshots=(StepSnapshot(index=0, scope_id="test::cp-output-owner", step=step),), + ) + provider = CellProfilerInvocationContractProviderFactory.provider_for_session( + session + ) + authored = next(normalize_function_pattern(step.func).iter_items()) + contract = provider.plans[(0, authored.key)].contract + assert contract.artifact_output_policy is CellProfilerRecordedArtifactOutputPolicy + (measurement,) = contract.artifact_outputs.of_artifact_type( + MeasurementsArtifactType + ) + assert measurement.measurement_feature_owner is IdentifyPrimaryObjectsModule + assert ArtifactSpecRelation.measurement_subject_for_output(measurement) is None + + compiled = compile_function_pattern( + step.func, + {}, + {}, + invocation_contract_provider=provider, + step_context=ArtifactDeclarationStepContext( + step_index=0, source_bindings=step.source_bindings + ), + ) + assert compiled.default_group.invocations[0].contract is contract + + # The same declaration is not valid under the native return carrier's + # single-subject obligation. Only that obligation differs between owners. + native = replace( + contract, metadata=replace(contract.metadata, runtime_adapter=None) + ) + with pytest.raises(ValueError, match="no declared measurement subject"): + native.validate_artifact_output_declarations() diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index 3d9f46d9de..b0f2936281 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -41,6 +41,7 @@ ) from openhcs.core.runtime_stores import RuntimeValueStore from openhcs.core.runtime_measurements import MeasurementRowAxisField, MeasurementTable +from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.runtime_adapters import ( RuntimeAdapterRequest, runtime_adapter, @@ -1895,7 +1896,7 @@ def test_module_runtime_adapter_records_declared_outputs_and_returns_main_flow( "runtime", lambda _request: object(), manages_artifact_inputs=True, - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) @artifact_outputs(ArtifactSpec.output("ModuleImage", ImageArtifactType)) def module_step(image, *, runtime): diff --git a/tests/unit/test_function_patterns.py b/tests/unit/test_function_patterns.py index 010385a2b9..e9da0944c6 100644 --- a/tests/unit/test_function_patterns.py +++ b/tests/unit/test_function_patterns.py @@ -13,6 +13,7 @@ ArtifactSpec, ArtifactSpecCollection, ArtifactType, + ArtifactMeasurementSubjectRelation, GroupLineageSourceRelation, ImageArtifactType, ObjectLabelsArtifactType, @@ -50,6 +51,7 @@ extract_artifact_declarations, normalize_pattern, ) +from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.runtime_adapters import runtime_adapter from openhcs.core.runtime_object_labels import ObjectLabelValue from openhcs.core.runtime_relationships import ObjectRelationshipDeclaration @@ -683,8 +685,9 @@ def test_component_projection_uses_compiled_per_group_source_lineage() -> None: measurements = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, - relations=tuple( - GroupLineageSourceRelation(source=source.ref()) for source in (blue, green) + relations=( + ArtifactMeasurementSubjectRelation(), + *(GroupLineageSourceRelation(source=source.ref()) for source in (blue, green)), ), ) @@ -693,7 +696,7 @@ def test_component_projection_uses_compiled_per_group_source_lineage() -> None: @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def measure_channels(image, *, runtime): del runtime @@ -757,11 +760,14 @@ def measure_channels(image, *, runtime): def test_unscoped_active_output_retains_complete_compiled_invocation_inputs() -> None: source = ArtifactSpec.input("OrigBlue", ImageArtifactType) - aggregate = ArtifactSpec.output("Aggregate", MeasurementsArtifactType) + aggregate = ArtifactSpec.output( + "Aggregate", MeasurementsArtifactType, + relations=(ArtifactMeasurementSubjectRelation(),), + ) scoped = ArtifactSpec.output( "BlueMeasurements", MeasurementsArtifactType, - relations=(GroupLineageSourceRelation(source=source.ref()),), + relations=(GroupLineageSourceRelation(source=source.ref()), ArtifactMeasurementSubjectRelation()), ) @artifact_inputs(source) @@ -769,7 +775,7 @@ def test_unscoped_active_output_retains_complete_compiled_invocation_inputs() -> @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def measure_with_aggregate(image, *, runtime): del runtime @@ -831,12 +837,12 @@ def test_shared_output_plan_uses_each_invocation_declared_group_lineage() -> Non blue_measurements = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, - relations=(GroupLineageSourceRelation(source=blue.ref()),), + relations=(GroupLineageSourceRelation(source=blue.ref()), ArtifactMeasurementSubjectRelation()), ) green_measurements = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, - relations=(GroupLineageSourceRelation(source=green.ref()),), + relations=(GroupLineageSourceRelation(source=green.ref()), ArtifactMeasurementSubjectRelation()), ) @artifact_inputs(blue) @@ -844,7 +850,7 @@ def test_shared_output_plan_uses_each_invocation_declared_group_lineage() -> Non @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def measure_blue(image, *, runtime): del runtime @@ -855,7 +861,7 @@ def measure_blue(image, *, runtime): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def measure_green(image, *, runtime): del runtime @@ -1162,13 +1168,16 @@ def consume_cross_channel(image, *, runtime): def test_adapter_managed_outputs_use_exact_compiled_output_plans(): cells_spec = ArtifactSpec.output("Cells", ObjectLabelsArtifactType) - measurements_spec = ArtifactSpec.output("Measurements", MeasurementsArtifactType) + measurements_spec = ArtifactSpec.output( + "Measurements", MeasurementsArtifactType, + relations=(ArtifactMeasurementSubjectRelation(),), + ) @artifact_outputs(cells_spec, measurements_spec) @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def runtime_recorded_outputs(image, *, runtime): return image @@ -1205,7 +1214,7 @@ def runtime_recorded_outputs(image, *, runtime): adapter = invocation.contract.runtime_adapter assert adapter is not None - assert adapter.manages_artifact_outputs + assert adapter.artifact_output_policy.records_outputs assert invocation.artifact_output_plans == (cells_output, measurement_output) assert selected_outputs == { cells_output.ref(): cells_output, diff --git a/tests/unit/test_function_runtime_source_projection.py b/tests/unit/test_function_runtime_source_projection.py index 8d6cd91d6a..bba81c6976 100644 --- a/tests/unit/test_function_runtime_source_projection.py +++ b/tests/unit/test_function_runtime_source_projection.py @@ -43,6 +43,7 @@ special_inputs, ) from openhcs.core.pipeline.path_planner import PathPlanner, PathPlannerArtifactStage +from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.runtime_adapters import runtime_adapter from openhcs.core.runtime_image_values import ( ImagePayloadMetadata, @@ -2453,7 +2454,7 @@ def test_runtime_chain_skips_adapter_invocation_without_component_outputs( @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) @artifact_outputs(first_spec) def record_first_labels(image, *, runtime): @@ -2463,7 +2464,7 @@ def record_first_labels(image, *, runtime): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) @artifact_outputs(second_spec) def record_second_labels(image, *, runtime): @@ -3229,7 +3230,7 @@ def test_adapter_recorded_outputs_use_compiled_canonical_context() -> None: @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) @artifact_outputs(outline_spec, first_labels_spec, second_labels_spec) def record_mixed_outputs(image, *, runtime): diff --git a/tests/unit/test_function_step_execution_plan.py b/tests/unit/test_function_step_execution_plan.py index 463fac7578..7c794bb17b 100644 --- a/tests/unit/test_function_step_execution_plan.py +++ b/tests/unit/test_function_step_execution_plan.py @@ -33,6 +33,7 @@ compile_function_pattern, ) from openhcs.core.pipeline.function_contracts import artifact_inputs, artifact_outputs +from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.runtime_adapters import RuntimeAdapterRequest, runtime_adapter from openhcs.core.runtime_output_matching import RuntimeReturnedOutputMatcher from openhcs.core.step_dependencies import ( @@ -461,7 +462,7 @@ def _cross_channel_output_invocation(): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def publish_channels(image, *, runtime): del runtime @@ -521,7 +522,7 @@ def test_compiler_handoff_preserves_exact_same_name_output_types(): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def publish_shared_outputs(image, *, runtime): del runtime @@ -616,7 +617,7 @@ def test_invocation_component_selection_projects_relation_owned_inputs(): "runtime", lambda _request: object(), manages_artifact_inputs=True, - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def publish_derived_channels(image, *, runtime): del runtime diff --git a/tests/unit/test_path_planner_materialization.py b/tests/unit/test_path_planner_materialization.py index b0906403dd..ef9a824836 100644 --- a/tests/unit/test_path_planner_materialization.py +++ b/tests/unit/test_path_planner_materialization.py @@ -7,6 +7,7 @@ from openhcs.constants.constants import AllComponents, GroupBy, VariableComponents from openhcs.constants.input_source import InputSource from openhcs.core.artifacts import ( + ArtifactMeasurementSubjectRelation, ArtifactInputPlan, ArtifactOutputPlan, ArtifactSidecarRole, @@ -76,6 +77,7 @@ PathPlannerValidationStage, ) from openhcs.core.pipeline.step_snapshot import StepSnapshot +from openhcs.core.artifact_key_selection import AdapterRecordedArtifactOutputPolicy from openhcs.core.runtime_adapters import runtime_adapter from openhcs.core.runtime_object_labels import ObjectLabelValue from openhcs.core.runtime_stores import RuntimeArtifactBatch @@ -2322,12 +2324,12 @@ def test_planner_derived_group_lineage_selects_exact_managed_invocation(): blue_measurements = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, - relations=(GroupLineageSourceRelation(blue.ref()),), + relations=(GroupLineageSourceRelation(blue.ref()), ArtifactMeasurementSubjectRelation()), ) green_measurements = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, - relations=(GroupLineageSourceRelation(green.ref()),), + relations=(GroupLineageSourceRelation(green.ref()), ArtifactMeasurementSubjectRelation()), ) @artifact_inputs(blue) @@ -2335,7 +2337,7 @@ def test_planner_derived_group_lineage_selects_exact_managed_invocation(): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def measure_blue(image, *, runtime): del runtime @@ -2346,7 +2348,7 @@ def measure_blue(image, *, runtime): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) def measure_green(image, *, runtime): del runtime @@ -3464,7 +3466,7 @@ def test_module_special_outputs_preserve_existing_main_flow_component_scopes(): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) @artifact_outputs(measurement_spec) def measurement_only(image, *, runtime): @@ -5340,7 +5342,7 @@ def test_main_input_dependency_skips_main_flow_preserving_steps(): @runtime_adapter( "runtime", lambda _request: object(), - manages_artifact_outputs=True, + artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) @artifact_outputs(measurement_spec) def measure(image, *, runtime): From 153f643ea5531e851714e9dc9c1cac6e6c580aed Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 16:41:19 -0400 Subject: [PATCH 09/17] Retain exact label-owned measurement group-scope regression --- ...easurement_contract_checkpoint_20260929.md | 23 +++- ...measurement_group_scope_triage_20260929.md | 85 +++++++++++++ .../unit/test_path_planner_materialization.py | 118 ++++++++++++++++++ 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 docs/plans/measurement_group_scope_triage_20260929.md diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 8e455bc999..302f384fb9 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -2,7 +2,8 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. -Base: `openhcsdev/main` at `283b21275`, fetched and merged normally on 2026-09-29. +Base: `openhcsdev/main` at `de23449a4`, fetched and merged normally on 2026-09-29 +through `21c15ad41`. The output-policy checkpoint used `283b21275`. The earlier checkpoint used `98d9b9d23`; the runtime checkpoint integrates new main through normal merge `9f5d444515`. No parent/main-worktree edits, rebases, resets, installed-source edits, or gitlink changes. @@ -10,6 +11,25 @@ resets, installed-source edits, or gitlink changes. Closes #133. References #204; its paired-channel and actual GUI acceptance remain open. This is a source-verified draft, not an installed/live-readiness claim. +## Frozen-harness compile incident: triaged, regression retained + +The original server5793 request `4f8e1d55-07f0-4bde-9f37-d24c0ea398f9` +failed at 20:24:35 UTC with `MeasureCells_4_measurements` group `"1"` having no +declared group-scope source. The original saved technical request's source hash +matches the server's `5fb62ebed47d`: it produces exact `Cells` under channel group +`"2"`, but explicitly dispatches its size/shape measurement under group `"1"`. +The real module already declares the exact labels-owned group lineage and object +subject. This is an invalid authored grouping, **not missing backend projection**. + +The new real-declaration/planner regression preserves that failure and proves +matching-group and declaration-derived controls compile: **3 passed**. Combined +with the output-policy review cases, **23 passed** in 1.87 seconds after current +main integration. No production relaxation, frozen replay or installation change +was made. Original request/variant identities, owner trace, pattern IDs and +acceptance boundaries are preserved in +`docs/plans/measurement_group_scope_triage_20260929.md`. This technical case stays +on #204/PR205; it does not resolve the separate paired runtime/live gate. + ## Implemented - Measurement output validation belongs to `MeasurementsArtifactType`. Compilation @@ -294,6 +314,7 @@ Draft PR: https://github.com/OpenHCSDev/openhcs/pull/205. - `tests/integration/test_measurement_declaration_journey.py` - `docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md` - `docs/plans/issue204_catalog_diagnosis_20260929.md` +- `docs/plans/measurement_group_scope_triage_20260929.md` Owned disposable test output was `/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929`, owner Zeno, purpose tiny synthetic plate and bounded pytest temporary outputs. diff --git a/docs/plans/measurement_group_scope_triage_20260929.md b/docs/plans/measurement_group_scope_triage_20260929.md new file mode 100644 index 0000000000..0082e61c18 --- /dev/null +++ b/docs/plans/measurement_group_scope_triage_20260929.md @@ -0,0 +1,85 @@ +# Measurement group-scope compile incident (#204 / PR205) + +Owner: Zeno. Classification: invalid authored grouping, not a missing backend +scope projection. This technical triage adds regression evidence to PR205; it +does not close #204's separate paired-channel runtime/live acceptance. + +## Actual request provenance + +- Frozen installed harness: `main` at `283b21275`, unchanged by this worker. +- Server 5793, job `4f8e1d55-07f0-4bde-9f37-d24c0ea398f9`, terminal failure + 2026-09-29 20:24:35 UTC (16:24:35 in the local log). +- Authorized log: + `/home/ts/wt/openhcs-h003-uncoached-20260929/output/runtime/xdg-data/openhcs/logs/openhcs_zmq_server_port_5793_1790712796745242.log`. + Request line 799 records `compile_only=True`, seven steps, + `pipeline_sha=5fb62ebed47d`, `request_sig=48d0e5ebf4c4`; lines 845–846 + record the terminal compile error, before measurement execution. +- Original technical request source: + `/home/ts/wt/openhcs-h003-uncoached-20260929/output/trial03/PipelineDocument.py`. + Its SHA256 is + `5fb62ebed47d9ed0f11f4499c26a96ac7107e655f47df64c7a51124cdf03f88d`, + matching the original server request. AST inspection extracted only declaration + identities and processing scopes, not scientific settings or outputs. +- Global scope declares `GroupBy.CHANNEL` and variable `SITE`. `Cells` is + explicitly produced under group `"2"`; `MeasureCells` explicitly dispatches + `measure_object_size_shape` under group `"1"`, selecting exact `"Cells"` + through the existing `select_object_sets_to_measure` binding. +- The separately saved `trial04/rejected-grouping.PipelineDocument.py` has hash + `09c3b345c49f...`, matching later job + `c5005087-f638-4a85-a8fb-abe4149e9df6`, not the original request. Its technical + compile receipt preserves the same error. It is corroboration, not a substitute + for the original request identity. + +## Owner and projection evidence + +`ObjectLabelDrivenPrimaryImageInputPolicy.invocation_domain_inputs` makes the +exact selected labels input the size/shape invocation domain. +`MeasurementArtifactOutputModule.measurement_output_relations` already emits +`GroupLineageSourceRelation(Cells input ref)`; `ObjectMeasurementInputModule` +also emits the object measurement subject. No declaration field is missing. + +`ArtifactProducer.has_explicit_invocation_group_ownership` preserves authored +grouped dispatch. `PathPlannerArtifactStage.output_groups_from_declared_relations` +therefore preserves output group `"1"`. The exact labels source has compiled +group `"2"`, so `output_group_scope_sources_by_group` correctly rejects it with: + +```text +MeasureCells_4_measurements ... group '1' has no declared group-scope source. +``` + +The relevant planner, producer, module-contract, module-relation and primary-image +policy files are unchanged from frozen `283b21275`. No backend scope relaxation, +new selector, registry, adapter exemption, or global channel exception is needed. +The separate #204 secondary-input runtime repair remains intact. + +## Focused regression and acceptance boundary + +`test_real_object_measurement_preserves_selected_labels_group_scope` reconstructs +the real `MeasureObjectSizeShapeModule` public invocation, canonical module +numbering and declaration contract, then runs actual artifact graph extraction, +planner map compilation and `compile_function_pattern`. The tiny synthetic +producer uses the exact `Cells` identity and channel group `"2"`: + +- Explicit measurement group `"1"` reproduces the original terminal error. +- Changing only that authored group to `"2"` compiles, preserving the exact label + input, object subject and output-to-label group-scope mapping. +- A non-dict invocation derives group `"2"` from the declaration-owned labels + domain. Its exact selector is consumed, not forwarded as a runtime kwarg. + +Three cases passed (99 deselected), 1.99 seconds. Combined with the output-policy +review regressions, **23 passed**, 1.87 seconds, after normal main integration +`21c15ad41` over `de23449a4`. All nine OpenHCS/dependency imports resolved into +this worktree using the existing shared venv and recorded submodule src paths. +The resource guard reported 10.2 GiB available RAM and a historical-swap warning; +only lightweight provider-free checks proceeded. No replay, MCP/JVM/GUI startup, +installation change, held-out/reference inspection or author message occurred. + +NRA/refactor-audit coverage is focused, not a complete semantic/proof scan: +IDEN-1/7 keeps authored dispatch, selected producer and measurement subject +distinct; BOUND-2 uses the existing typed owners; MEMB-1/2 introduces no roster; +IMPL-4/10 preserves declaration-driven compile rejection before execution. +The three new cases require **zero production edits**. Tests and this receipt +are the only new scope-specific changes. Actual installed paired-channel and GUI +acceptance remains pending the coordinator's serialized validation handoff after +Confucius releases the frozen runtime. No new issue is needed for this authored +contract error; the case is retained on existing #204 / PR205. diff --git a/tests/unit/test_path_planner_materialization.py b/tests/unit/test_path_planner_materialization.py index ef9a824836..17723ee62e 100644 --- a/tests/unit/test_path_planner_materialization.py +++ b/tests/unit/test_path_planner_materialization.py @@ -19,6 +19,7 @@ InputGroupLineageSourceRelation, InputStackBroadcastSourceRelation, ObjectLabelsArtifactType, + ObjectMeasurementSubjectRelation, MeasurementsArtifactType, RelationshipsArtifactType, SpecialArtifactType, @@ -2418,6 +2419,123 @@ def measure_green(image, *, runtime): } +@pytest.mark.parametrize("measurement_group", ["1", "2", DEFAULT_GROUP_KEY]) +def test_real_object_measurement_preserves_selected_labels_group_scope( + measurement_group, +): + """An exact label selector does not rewrite an explicitly authored group.""" + from openhcs.core.function_patterns import normalize_function_pattern + from openhcs.interop.cellprofiler.compile_time_contracts import ( + CellProfilerInvocationContractProvider, + ) + from openhcs.processing.backends.cellprofiler.shape import ( + MeasureObjectSizeShapeModule, + measure_object_size_shape, + ) + + planner = _artifact_planner_stub() + labels_plan = _record_declared_output( + planner, + ArtifactOutputPlan( + name="Cells", + path="/memory/Cells.pkl", + artifact_type=ObjectLabelsArtifactType, + group_keys=("2",), + group_component=AllComponents.CHANNEL, + variable_components=(AllComponents.SITE,), + paths_by_group={"2": "/memory/Cells_2.pkl"}, + ), + ) + selector = ( + MeasureObjectSizeShapeModule.object_measurement_binding.require_parameter_name() + ) + invocation_pattern = (measure_object_size_shape, {selector: "Cells"}) + pattern = ( + invocation_pattern + if measurement_group == DEFAULT_GROUP_KEY + else {measurement_group: [invocation_pattern]} + ) + snapshot = _snapshot(name="MeasureCells", func=pattern) + labels_input = ArtifactSpec.input("Cells", ObjectLabelsArtifactType) + step_context = ArtifactDeclarationStepContext( + step_name=snapshot.step.name, + step_index=snapshot.index, + group_by=GroupBy.CHANNEL, + available_artifacts=ArtifactSpecCollection((labels_input,)), + available_artifact_producers=( + ArtifactProducer( + ArtifactSpec.output("Cells", ObjectLabelsArtifactType), + groups=labels_plan.group_keys, + invocation_keys=(), + producer_step_index=2, + ), + ), + ) + authored = next(normalize_function_pattern(pattern).iter_items()) + blocks, consumed_names = MeasureObjectSizeShapeModule.module_blocks_for_invocation( + invocation=authored, + step_context=step_context, + ) + (numbered_blocks,), _ = MeasureObjectSizeShapeModule.number_step_invocation_blocks( + (blocks,), first_module_num=4 + ) + contract, consumed_names = MeasureObjectSizeShapeModule.invocation_callable_contract( + invocation=authored, + numbered_module_blocks=numbered_blocks, + consumed_kwarg_names=consumed_names, + step_context=step_context, + ) + provider = CellProfilerInvocationContractProvider( + {(snapshot.index, authored.key): InvocationContractPlan(contract, consumed_names)} + ) + declarations = extract_artifact_declarations( + pattern, + invocation_contract_provider=provider, + step_context=step_context, + ) + (measurement,) = contract.artifact_outputs.of_artifact_type(MeasurementsArtifactType) + (object_input,) = contract.artifact_inputs.of_artifact_type(ObjectLabelsArtifactType) + assert object_input.ref() == labels_input.ref() + assert object_input.parameter_name == "labels" + assert measurement.measurement_feature_owner is MeasureObjectSizeShapeModule + assert measurement.name == "MeasureCells_4_measurements" + assert measurement.group_scope_sources() == (labels_input.ref(),) + assert ( + MeasurementsArtifactType.require_output_subject(measurement) + == ObjectMeasurementSubjectRelation(labels_input.ref()).measurement_subject() + ) + assert selector in consumed_names + group_scope = PathPlannerGroupScope.from_raw( + ("1", "2"), component=AllComponents.CHANNEL + ) + + if measurement_group == "1": + with pytest.raises( + ValueError, + match="MeasureCells_4_measurements.*group '1' has no declared group-scope source", + ): + planner.artifacts.compile_plan_maps(snapshot, 3, declarations, group_scope) + return + + maps = planner.artifacts.compile_plan_maps(snapshot, 3, declarations, group_scope) + measurement_plan = maps.outputs[measurement.ref()] + assert maps.inputs[labels_input.ref()].group_keys == labels_plan.group_keys + assert measurement_plan.group_keys == ("2",) + assert measurement_plan.group_scope_sources_by_group == { + "2": (labels_input.ref(),), + } + compiled = compile_function_pattern( + pattern, + maps.inputs, + maps.outputs, + invocation_contract_provider=provider, + step_context=step_context, + ) + (invocation,) = tuple(compiled.iter_invocations()) + assert selector not in dict(invocation.kwargs) + assert invocation.contract is contract + + def test_declared_group_lineage_cannot_rewrite_scalar_step_execution_scope(): planner = _artifact_planner_stub() _record_declared_output( From 94595f2719d0415c7e16f9eecddac0022d9160b6 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 22:22:46 -0400 Subject: [PATCH 10/17] Strengthen subject journey with typed feature owner and source lineage --- ...easurement_contract_checkpoint_20260929.md | 71 ++++++++++++++++++- .../test_measurement_declaration_journey.py | 49 +++++++++++-- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 302f384fb9..a0ef8a66ee 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -2,8 +2,10 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. -Base: `openhcsdev/main` at `de23449a4`, fetched and merged normally on 2026-09-29 -through `21c15ad41`. The output-policy checkpoint used `283b21275`. +Base: `openhcsdev/main` at `589c33a12`, merged normally through `684e4bbd3`. +Remote main was independently rechecked at that same revision. The preceding +checkpoint merged `de23449a4` through `21c15ad41`. +The output-policy checkpoint used `283b21275`. The earlier checkpoint used `98d9b9d23`; the runtime checkpoint integrates new main through normal merge `9f5d444515`. No parent/main-worktree edits, rebases, resets, installed-source edits, or gitlink changes. @@ -11,6 +13,71 @@ resets, installed-source edits, or gitlink changes. Closes #133. References #204; its paired-channel and actual GUI acceptance remain open. This is a source-verified draft, not an installed/live-readiness claim. +## Current-main subject journey: prepared; refreshed live gate pending + +The newly reported frozen-author failure is precisely #133: a typed `PURE_3D` +measurement output can carry a feature owner and source lineage without declaring +what it measures. The original attempt remains untouched. No author source, +science, pixels, parameters or references were needed or inspected for this +continuation; no replay or author coaching occurred. + +The existing synthetic integration journey now declares its own `CountFeature` +and nominal feature owner, CSV materialization and source-stack lineage in both +variants. The corrected declaration is a `dataclasses.replace` of the invalid +one, adding **only** `ImageMeasurementSubjectRelation(CountedImage.ref())`. +Both variants render/parse through `PipelineDocumentAuthority`; the test asserts +the reconstructed feature owner and source relation before the real headless +compile boundary. The invalid callable still raises if entered. The valid case +uses the real compiler and orchestrator and checks both image and measurement +outputs and four counted synthetic pixels. These revised integration assertions +are prepared, **not yet rerun** at current main. + +Actual lightweight checks at this checkpoint: + +- All nine imported package paths resolve inside this worktree, using the required + existing interpreter and eight recorded submodule `src` paths. PolyStore + `1209068f5` and ZMQRuntime `0f9e840a9` were synchronized from existing local Git + objects only: no dependency download, donor source edit or installation change. +- Four direct, real output-policy checks passed: native-return and ordinary + adapter-recorded owners each reject a missing subject despite the declared + feature owner and source lineage, then accept the subject-only correction. + This is owner-hook behavior evidence, **not** compile/execution/CSV evidence. +- AST parse and in-memory Python compilation passed for all **24 changed Python + files** against current main; `git diff --check` passed. Focused guards verify + common kind validation precedes payload-policy validation with no exemption + branch, and the removed `manages_artifact_outputs` has no source/test callers. +- The original exact two-producer paired execution remains enabled, together + with omitted-selector ambiguity/single-producer cases and real unscoped and + asymmetric `RuntimeArtifactInput` regressions. No assertion, channel check, + selector, source policy or production owner was weakened in this continuation. + +NRA/refactor-audit coverage is **focused source/caller/AST inspection**, not a +complete NRA scan or native proof. Actual owner evidence: IMPL-1/4/10 puts output +invariants on the artifact kind and native/recorded obligations on their declared +policy; the compiler calls `ArtifactPlanKeySelector` and runtime reuses +`MeasurementsArtifactType.require_output_subject`. IDEN-1 keeps feature ownership, +source lineage and measurement subject independent; schema/CSV is not a subject. +MEMB-1/2 and IDEN-5 derive the catalog selector from the existing binding's +`require_parameter_name`, with no copied roster/store. IDEN-7/BOUND-2 keeps exact +producer/group selection before typed contextual compatibility; empty/asymmetric +additional constraints do not alter global source-image matching semantics. +The retained new-kind experiment declares one new artifact kind, requires +**zero compiler/consumer changes**, and covers all three output policies, +independent input management and both contract/compile boundaries. Its prior +20-case passing evidence, including legitimate heterogeneous CP ownership, is +historical below, not relabeled as a current-main rerun. + +Remaining named dependency: the parent integration owner must explicitly release +the serialized slot after the blind freeze and review the tiny synthetic native +compile/execute gate. This worktree lacks the current-main `_tabular_native` +extension (`find_spec` returns `None`); no native build, environment startup, +MCP/JVM/GUI, installation or managed-skill change was attempted. Slot release +must not be inferred merely from the original 02:32:25 UTC deadline passing. +After release, validate the two subject variants first, then the unweakened full +six-case declaration journey and focused paired-source cases. Installed/live +verification and merging remain with the parent; optional CI or broader paired +research is not a prerequisite for shipping the reviewed #133 checkpoint. + ## Frozen-harness compile incident: triaged, regression retained The original server5793 request `4f8e1d55-07f0-4bde-9f37-d24c0ea398f9` diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index 3106b44c05..bc4cb04996 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace import numpy as np import pytest @@ -24,6 +24,7 @@ MainFlowStackOutputSpec, MeasurementsArtifactType, ObjectLabelsArtifactType, + SourceStackLineageSourceRelation, ) from openhcs.core.config import ( GlobalPipelineConfig, @@ -41,6 +42,10 @@ from openhcs.core.pipeline_document import PipelineDocumentAuthority from openhcs.core.pipeline.function_contracts import artifact_outputs from openhcs.core.runtime_object_labels import object_label_dense_array +from openhcs.core.runtime_measurements import ( + RuntimeMeasurementFeature, + RuntimeMeasurementFeatureOwner, +) from openhcs.core.source_bindings import ( ComponentSelector, NamedSourceBinding, @@ -82,15 +87,35 @@ class CountRow: pixel_count: int +class CountFeature(RuntimeMeasurementFeature): + PIXEL_COUNT = "pixel_count" + + +class CountFeatureOwner(RuntimeMeasurementFeatureOwner): + feature_type = CountFeature + + @classmethod + def owns_measurement_feature_name(cls, feature_name): + return any(feature.value == feature_name for feature in cls.feature_type) + + @classmethod + def owns_primary_measurement_feature_name(cls, feature_name): + return cls.owns_measurement_feature_name(feature_name) + + COUNT_IMAGE = MainFlowStackOutputSpec.output("CountedImage", ImageArtifactType) MISSING_SUBJECT_ROWS = ArtifactSpec.output( "PixelCounts", MeasurementsArtifactType, materialization=MaterializationSpec(CsvOptions()), + measurement_feature_owner=CountFeatureOwner, + relations=(SourceStackLineageSourceRelation(COUNT_IMAGE.ref()),), ) -IMAGE_SUBJECT_ROWS = ArtifactSpec.output( - "PixelCounts", MeasurementsArtifactType, - materialization=MaterializationSpec(CsvOptions()), - relations=(ImageMeasurementSubjectRelation(COUNT_IMAGE.ref()),), +IMAGE_SUBJECT_ROWS = replace( + MISSING_SUBJECT_ROWS, + relations=( + *MISSING_SUBJECT_ROWS.relations, + ImageMeasurementSubjectRelation(COUNT_IMAGE.ref()), + ), ) @@ -341,6 +366,18 @@ def test_headless_entrypoint_requires_subject_and_executes_corrected_rows(tmp_pa "Count pixels", {}, )], ) + document = PipelineDocumentAuthority.from_source( + PipelineDocumentAuthority.render(document) + ) + invocation = next( + normalize_function_pattern(document.pipeline_steps[0].func).iter_items() + ) + [rows] = ( + spec for spec in invocation.contract.artifact_specs.specs + if spec.artifact_type is MeasurementsArtifactType + ) + assert rows.measurement_feature_owner is CountFeatureOwner + assert SourceStackLineageSourceRelation(COUNT_IMAGE.ref()) in rows.relations global_config = GlobalPipelineConfig(num_workers=1, use_threading=True) if not valid: with pytest.raises(ValueError, match="PixelCounts.*no declared measurement subject") as exc: @@ -352,5 +389,7 @@ def test_headless_entrypoint_requires_subject_and_executes_corrected_rows(tmp_pa assert results["A01"].is_success(), results["A01"].error_message store = bundle.runtime_contexts["A01"].runtime_value_store [counts] = store.find(name="PixelCounts", axis_id="A01") + [image] = store.find(name="CountedImage", axis_id="A01") + assert image.key.artifact_type is ImageArtifactType assert counts.value.data.subject.source_image_name == "CountedImage" assert tuple(counts.value.data.rows.column_values("pixel_count")) == (4,) From 3dfa616a8e6330e444bf8b5fbdc0d9fc84425420 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 22:59:27 -0400 Subject: [PATCH 11/17] Bind measurement journey lineage through existing main-flow output owner --- ...easurement_contract_checkpoint_20260929.md | 82 +++++++++++++++---- .../test_measurement_declaration_journey.py | 27 ++++-- 2 files changed, 89 insertions(+), 20 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index a0ef8a66ee..fb5894a076 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -2,9 +2,9 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. -Base: `openhcsdev/main` at `589c33a12`, merged normally through `684e4bbd3`. -Remote main was independently rechecked at that same revision. The preceding -checkpoint merged `de23449a4` through `21c15ad41`. +Base: `openhcsdev/main` at `32d070c26` (merged PR255), integrated normally through +`85d8b2ae7`. The preceding checkpoint merged `589c33a12` through `684e4bbd3`, +and `de23449a4` through `21c15ad41` before that. The output-policy checkpoint used `283b21275`. The earlier checkpoint used `98d9b9d23`; the runtime checkpoint integrates new main through normal merge `9f5d444515`. No parent/main-worktree edits, rebases, @@ -21,17 +21,57 @@ what it measures. The original attempt remains untouched. No author source, science, pixels, parameters or references were needed or inspected for this continuation; no replay or author coaching occurred. -The existing synthetic integration journey now declares its own `CountFeature` -and nominal feature owner, CSV materialization and source-stack lineage in both -variants. The corrected declaration is a `dataclasses.replace` of the invalid +The existing synthetic integration journey declares its own `CountFeature` +and nominal feature owner, CSV materialization and `MainFlowStackOutputSpec` +for both row variants. The existing `MainFlowArtifactContractProvider` binds +their stack/group lineage to the actual current image **input**. The corrected +declaration is a `dataclasses.replace` of the invalid one, adding **only** `ImageMeasurementSubjectRelation(CountedImage.ref())`. Both variants render/parse through `PipelineDocumentAuthority`; the test asserts -the reconstructed feature owner and source relation before the real headless -compile boundary. The invalid callable still raises if entered. The valid case +the reconstructed feature owner and real provider's input-qualified scope before +the headless compile boundary; the subject remains the exact image **output** +ref. The invalid callable still raises if entered. The valid case uses the real compiler and orchestrator and checks both image and measurement outputs and four counted synthetic pixels. These revised integration assertions are prepared, **not yet rerun** at current main. +### Parent review: output-qualified lineage fixture repaired + +The parent correctly identified an independent invalid declaration in checkpoint +`94595f271`: generic `ArtifactSpec.output` with +`SourceStackLineageSourceRelation(CountedImage output ref)` does not make a legal +group-scope input. The earlier four output-policy probes did not exercise that +boundary. Their passing result was not evidence that this valid-subject variant +could reach execution or CSV. No such readiness is claimed. + +A bounded source-only probe using the actual `CallableContract`, +`NormalizedFunctionItem`, `ArtifactDeclarationStepContext` and +`MainFlowArtifactContractProvider` reproduced the failure: +`Callable 'count' output group-scope relations reference undeclared inputs` +with the exact `CountedImage` OUTPUT ref. Switching both variants to the existing +`MainFlowStackOutputSpec` yields three observed outcomes in the final probe: + +1. The original generic output-qualified lineage still fails the real + `CallableContract.group_scope_inputs` boundary. +2. The corrected input-qualified lineage passes that boundary while its missing + measurement subject still fails output-owner validation. +3. Adding only `ImageMeasurementSubjectRelation(CountedImage.ref())` passes both + boundaries and retains the exact OUTPUT subject separately from INPUT lineage. + +The first probe confirmed the red witness, then used a nonexistent +`ArtifactSpec.measurement_subject()` test API. That probe error was corrected to +`MeasurementsArtifactType.require_output_subject`; it was not a runtime defect. +The final probe passed in 0.58 seconds without invoking the callable. It uses +minimal metadata, not CSV/row/native execution, and is explicitly source-contract +evidence. The actual integration fixture retains its feature owner, CSV options, +roundtrip and compile/execute assertions for the released-slot gate. + +NRA/refactor-audit finding: IDEN-1 separates invocation source/group ownership +from measurement subject; IMPL-4 reuses the output declaration's existing binding +hook instead of restoring consumer inference. Only the journey fixture and this +receipt are edited. No guard relaxation, new owner/registry, parent PR255 file +edit, source-binding edit, installed-root use or production-code change occurred. + Actual lightweight checks at this checkpoint: - All nine imported package paths resolve inside this worktree, using the required @@ -67,16 +107,28 @@ independent input management and both contract/compile boundaries. Its prior 20-case passing evidence, including legitimate heterogeneous CP ownership, is historical below, not relabeled as a current-main rerun. -Remaining named dependency: the parent integration owner must explicitly release -the serialized slot after the blind freeze and review the tiny synthetic native -compile/execute gate. This worktree lacks the current-main `_tabular_native` -extension (`find_spec` returns `None`); no native build, environment startup, -MCP/JVM/GUI, installation or managed-skill change was attempted. Slot release -must not be inferred merely from the original 02:32:25 UTC deadline passing. -After release, validate the two subject variants first, then the unweakened full +The parent explicitly granted the finite source-live slot after freezing and +closing the original blind runtime and its finite installed acceptance attempt. +The worker acquired `validation.lock` nonblocking for the admission check. +It exited **78**: available RAM **14.08 GiB** exceeds the initial 11 GiB floor, +but `/home` available disk **19.7195 GiB** is below the required 20 GiB floor. +The general resource guard's rounded 20.0 GiB is not an admission proof. +No build/test/MCP/JVM/GUI process started; the admission process exited and +released its lock. Previously recorded worker-owned disposable scratch is +already empty (4 KiB directory), so no other owner's artifacts were removed. + +Remaining named dependency: parent resource admission needs at least 0.3 GiB +additional available disk plus build margin. This worktree still lacks the +current-main `_tabular_native` extension (`find_spec` returns `None`). No native +build, environment startup, installation or managed-skill change occurred. +After admission, validate the two subject variants first, then the unweakened full six-case declaration journey and focused paired-source cases. Installed/live verification and merging remain with the parent; optional CI or broader paired research is not a prerequisite for shipping the reviewed #133 checkpoint. +Parent owns the separate #254 payload ABI/PURE_2D follow-up, including only +`FunctionChainInvocationExecutor.main_flow_call_argument` in the shared runtime +file. This worker's measurement strategy/input-loader scope does not collide; +no cherry-pick of that unpublished follow-up or installed-root validation occurs. ## Frozen-harness compile incident: triaged, regression retained diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index bc4cb04996..1529af82e0 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -18,13 +18,12 @@ from openhcs.core.artifacts import ( ArtifactInputPlan, ArtifactOutputPlan, - ArtifactSpec, + ArtifactSpecCollection, ImageArtifactType, ImageMeasurementSubjectRelation, MainFlowStackOutputSpec, MeasurementsArtifactType, ObjectLabelsArtifactType, - SourceStackLineageSourceRelation, ) from openhcs.core.config import ( GlobalPipelineConfig, @@ -33,6 +32,10 @@ PipelineConfig, ) from openhcs.core.function_patterns import normalize_function_pattern +from openhcs.core.invocation_artifacts import ( + ArtifactDeclarationStepContext, + MainFlowArtifactContractProvider, +) from openhcs.core.function_step_document import FunctionStepDocumentAuthority from openhcs.core.measurement_feature_queries import measurement_values_for_feature from openhcs.core.measurement_row_materialization import DataclassMeasurementColumnarRows @@ -104,11 +107,10 @@ def owns_primary_measurement_feature_name(cls, feature_name): COUNT_IMAGE = MainFlowStackOutputSpec.output("CountedImage", ImageArtifactType) -MISSING_SUBJECT_ROWS = ArtifactSpec.output( +MISSING_SUBJECT_ROWS = MainFlowStackOutputSpec.output( "PixelCounts", MeasurementsArtifactType, materialization=MaterializationSpec(CsvOptions()), measurement_feature_owner=CountFeatureOwner, - relations=(SourceStackLineageSourceRelation(COUNT_IMAGE.ref()),), ) IMAGE_SUBJECT_ROWS = replace( MISSING_SUBJECT_ROWS, @@ -377,7 +379,22 @@ def test_headless_entrypoint_requires_subject_and_executes_corrected_rows(tmp_pa if spec.artifact_type is MeasurementsArtifactType ) assert rows.measurement_feature_owner is CountFeatureOwner - assert SourceStackLineageSourceRelation(COUNT_IMAGE.ref()) in rows.relations + source_input = _source("DNA", "1").input_spec() + binding = MainFlowArtifactContractProvider()( + invocation, + ArtifactDeclarationStepContext( + main_flow_artifacts=ArtifactSpecCollection((source_input,)), + ), + ) + assert binding is not None + assert binding.contract.group_scope_inputs.specs == (source_input,) + bound_rows = binding.contract.artifact_outputs.by_ref(rows.ref()) + assert bound_rows is not None + assert bound_rows.source_stack_scope_sources() == (source_input.ref(),) + assert binding.contract.output_group_scope_sources == (source_input.ref(),) + assert ( + ImageMeasurementSubjectRelation(COUNT_IMAGE.ref()) in bound_rows.relations + ) is valid global_config = GlobalPipelineConfig(num_workers=1, use_threading=True) if not valid: with pytest.raises(ValueError, match="PixelCounts.*no declared measurement subject") as exc: From 4f43323681685f951299814268a92e51183d64b5 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Wed, 30 Sep 2026 00:10:36 -0400 Subject: [PATCH 12/17] Validate finalized input ABI before dependent output obligations --- ...easurement_contract_checkpoint_20260929.md | 182 +++++++++++++----- openhcs/core/function_patterns.py | 1 + openhcs/core/pipeline/path_planner.py | 1 - .../test_measurement_declaration_journey.py | 9 + tests/unit/test_artifact_output_ownership.py | 41 +++- .../unit/test_function_step_execution_plan.py | 2 +- .../unit/test_path_planner_materialization.py | 8 +- 7 files changed, 197 insertions(+), 47 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index fb5894a076..95f0e52fe4 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -10,10 +10,100 @@ The earlier checkpoint used `98d9b9d23`; the runtime checkpoint integrates new main through normal merge `9f5d444515`. No parent/main-worktree edits, rebases, resets, installed-source edits, or gitlink changes. -Closes #133. References #204; its paired-channel and actual GUI acceptance remain -open. This is a source-verified draft, not an installed/live-readiness claim. - -## Current-main subject journey: prepared; refreshed live gate pending +Closes #133. References #204; paired-channel headless execution now passes, but +actual GUI/installed acceptance remains open. This is a source-headless verified +draft, not an installed/live-readiness claim. + +## Current-main finite journey: terminal, working checkpoint + +The parent released the finite serial slot after its terminal PR259 attempt. +Every build/test phase acquired `validation.lock` nonblocking. Raw initial +headroom was **12.99 GiB available RAM / 24.90 GiB disk**, and final admission +was **13.07 GiB / 29.88 GiB**. Initial RAM >=11 GiB, trial RAM >=8 GiB and disk +>=20 GiB were enforced; only the authorized historical-swap warning was waived. +The existing interpreter and all nine imported source roots resolved inside this +worktree. `setup.py build_ext --inplace --parallel 1` completed in **2.59 seconds**, +building the two existing native extensions locally, without installation, +downloads or shared-source changes. + +Actual results, in order (counts overlap; they are not an additive unique total): + +- Subject-only variants: **2 passed, 4 deselected**, 3.61 seconds. The real + headless compiler rejects missing subject before callable entry. Adding only + the exact image subject compiles, executes and returns the image-owned rows. +- First complete journey: **1 failed, 5 passed**, 37.62 seconds. Actual paired + execution already passed. The omitted-selector case still failed closed, but + common output validation masked the original scalar-label ambiguity diagnostic. + Its original failure is retained in `full-six-case-journey.log`, SHA256 + `850195091fec271e2a8b43d6dc7ce29a365f1ae4a0339cb46398efee3d4ac175`. +- Runtime, output-policy, catalog and identity controls: **89 passed**, 3.37 + seconds. This includes exact unscoped/asymmetric admission and coordinate, + producer, relation and ambiguity negative controls. +- After the ordering repair, the unweakened complete journey: **6 passed**, 7.54 + seconds. A broader compile run exposed three incomplete older test declarations: + **3 failed, 269 passed**, 9.38 seconds. Original failures are retained in + `finalized-contract-controls.log`, SHA256 + `285082789885e46a8762cfe5913ccff507b68c244531e3f86ce84ee96262ff28`. +- Final combined run: **278 passed**, 15.59 seconds: all 272 cases in + `test_function_patterns.py`, `test_artifact_output_ownership.py`, + `test_function_step_execution_plan.py`, `test_path_planner_materialization.py` + and `test_function_artifact_outputs.py`, plus all six continuous cases in + `tests/integration/test_measurement_declaration_journey.py`. No journey case + was deselected or weakened. The corrected subject case now also reads the + actual emitted details CSV and asserts `pixel_count=4` and + `source_image_name=CountedImage` against the returned subject. + +The diagnostic repair moves the **existing** +`CallableContract.validate_artifact_input_parameter_bindings()` call from the +later path-planner consumer into `_compile_invocation`, immediately after +declaration finalization and before dependent output validation. The old call is +deleted. Three new policy-parametrized cases prove scalar `labels` ambiguity wins +at compile while the same declaration still independently fails the artifact +kind's conflicting-subject invariant. No validator, output-policy exemption, +selector, subject requirement or source compatibility predicate was weakened. + +The three older fixture repairs declare what their assertions already require: +the artifact-fed `illumination_function` parameter now exists on the callable; +both sidecar-conflict rows declare artifact subjects; and the channel-grouped +rows declare their exact source image as subject separately from group lineage. +Their component projection, sidecar conflict and channel/producer assertions are +unchanged. The final output-policy file has **23 passing cases**, including the +new-kind matrix across native, adapter-recorded and CP-recorded owners and the +legitimate heterogeneous CP recording contract. + +NRA/refactor-audit evidence is focused, not a complete NRA semantic/proof scan: +IDEN-1 keeps input lineage and output subject separate; BOUND-2/IMPL-4 retains +validation with the finalized declaration owner; IMPL-5/12 removes the replaced +consumer call in place. Adding an output kind still needs zero compiler/consumer +branches. The catalog still derives its selector from the binding owner, and +paired context still uses declared source relations rather than a global channel +exception. Current caller and AST inspection complements the real owner cases. +All **25 changed Python files** against the validated main32 parse and compile +in memory. The two current production deltas contain only one added owner call +and one removed call, with zero added `match` or `if` nodes. Caller search finds +one finalized compiler call and the pre-existing explicit authoring wrapper; no +remaining `manages_artifact_outputs` caller exists. `git diff --check` passes. +The first AST command compared the moving remote main and tried to read an +upstream-only new benchmark file absent from this older worktree; the check was +corrected to the actual validated main32 baseline, without editing that file. + +Build/test processes **2304493, 2304586, 2305892, 2306949, 2320857, 2321108, +2328819** and supervisors **2304492, 2305891, 2320856, 2328817** are all terminal +and absent. Final test/supervisor exit codes were zero; no worktree process or +surviving handle remains. The worker explicitly returned the native slot to the +parent. No MCP server, JVM or GUI was launched. This exercises the real compiler, +orchestrator, processing and CSV persistence on self-generated tiny fixtures; +it does not claim installed MCP/GUI acceptance or a replay of any blind attempt. + +Owned scratch/log root: +`/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929/pr205-live-20260930`. +Command logs, including both original failures and the final passing run, are +retained. Approximately 11 MiB of verified regeneratable build intermediates, +synthetic pytest outputs and caches were removed after process termination. +Locally built worktree extensions remain; no source, saved session, author input, +reference, installed package or other owner's output was removed. + +## Historical subject preparation and lineage-boundary review The newly reported frozen-author failure is precisely #133: a typed `PURE_3D` measurement output can carry a feature owner and source lineage without declaring @@ -32,8 +122,8 @@ the reconstructed feature owner and real provider's input-qualified scope before the headless compile boundary; the subject remains the exact image **output** ref. The invalid callable still raises if entered. The valid case uses the real compiler and orchestrator and checks both image and measurement -outputs and four counted synthetic pixels. These revised integration assertions -are prepared, **not yet rerun** at current main. +outputs and four counted synthetic pixels. These assertions and actual CSV +persistence now pass in the current-main finite journey above. ### Parent review: output-qualified lineage fixture repaired @@ -69,8 +159,9 @@ roundtrip and compile/execute assertions for the released-slot gate. NRA/refactor-audit finding: IDEN-1 separates invocation source/group ownership from measurement subject; IMPL-4 reuses the output declaration's existing binding hook instead of restoring consumer inference. Only the journey fixture and this -receipt are edited. No guard relaxation, new owner/registry, parent PR255 file -edit, source-binding edit, installed-root use or production-code change occurred. +receipt were edited in that preparation checkpoint. No guard relaxation, new +owner/registry, parent PR255 file edit, source-binding edit, installed-root use +or production-code change occurred in that checkpoint. Actual lightweight checks at this checkpoint: @@ -107,7 +198,7 @@ independent input management and both contract/compile boundaries. Its prior 20-case passing evidence, including legitimate heterogeneous CP ownership, is historical below, not relabeled as a current-main rerun. -The parent explicitly granted the finite source-live slot after freezing and +Historical first admission: the parent explicitly granted the slot after freezing and closing the original blind runtime and its finite installed acceptance attempt. The worker acquired `validation.lock` nonblocking for the admission check. It exited **78**: available RAM **14.08 GiB** exceeds the initial 11 GiB floor, @@ -117,19 +208,31 @@ No build/test/MCP/JVM/GUI process started; the admission process exited and released its lock. Previously recorded worker-owned disposable scratch is already empty (4 KiB directory), so no other owner's artifacts were removed. -Remaining named dependency: parent resource admission needs at least 0.3 GiB +At that admission, the named dependency was at least 0.3 GiB additional available disk plus build margin. This worktree still lacks the current-main `_tabular_native` extension (`find_spec` returns `None`). No native build, environment startup, installation or managed-skill change occurred. -After admission, validate the two subject variants first, then the unweakened full -six-case declaration journey and focused paired-source cases. Installed/live -verification and merging remain with the parent; optional CI or broader paired -research is not a prerequisite for shipping the reviewed #133 checkpoint. +The released-slot continuation above subsequently built the extensions and +validated both subject variants, the complete six-case journey and paired-source +controls. Installed/live verification and merging remain with the parent; +optional CI is not a prerequisite for shipping the reviewed #133 checkpoint. Parent owns the separate #254 payload ABI/PURE_2D follow-up, including only `FunctionChainInvocationExecutor.main_flow_call_argument` in the shared runtime file. This worker's measurement strategy/input-loader scope does not collide; no cherry-pick of that unpublished follow-up or installed-root validation occurs. +Finite-slot continuation after parent's terminal PR259 source-live attempt: +parent explicitly returned the slot to PR205. Worker-owned disposable scratch is +`/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929/pr205-live-20260930`, +purpose local native intermediates, command logs, and self-generated tiny pytest +plates/cache only. No source, saved session, author input or reference belongs +there. Keep failed-attempt receipts until classified, then remove verified +regeneratable build/test scratch after preserving results. The build uses the +existing `setup.py build_ext --inplace --parallel 1`; no installation or download. +Validate required imported paths and fresh RAM/disk floors under the nonblocking +shared lock before build, then both subject variants before the complete six-case +journey. No parent PR259 contextualization method is edited by this worker. + ## Frozen-harness compile incident: triaged, regression retained The original server5793 request `4f8e1d55-07f0-4bde-9f37-d24c0ea398f9` @@ -179,7 +282,7 @@ on #204/PR205; it does not resolve the separate paired runtime/live gate. Producer address/group selection, stack-plane projection, and ambiguity checks remain authoritative; the old duplicate fixed-coordinate loops are deleted. -## Focused evidence +## Historical focused evidence (current rerun above) Using `/home/ts/code/projects/openhcs/.venv/bin/python` with this worktree and all eight recorded submodule `src` directories on `PYTHONPATH`; all nine imported @@ -364,31 +467,22 @@ both compile and runtime use it unchanged. A new output kind supplies its own validation hook, with zero edits to the generic compiler consumer. The duplicate-resolution removal is factoring; no pure file move is claimed. -## Remaining acceptance - -- The original paired-channel journey failed **before measurement**, while - secondary consumes `Nuclei`: the exact record has fixed channel 1 scope, - but `RuntimeArtifactInput._matches_execution_scope` compares it with the - secondary source's channel 2 scope and rejects it. Address, axis, and producer - identity match. Reproduce with `pytest -q - tests/integration/test_measurement_declaration_journey.py -k paired --tb=short` - under the serialized validation lock after coordinator handoff. Zeno is the - active runtime-fix owner under the explicit scope extension. The focused fix is - present and store regressions pass, but complete paired execution remains - unverified. No `core/source_bindings.py`, `core/source_binding_workspace.py`, - source preparation, ROI, viewer, or #211 authorization method/test was modified. -- Scheduling: parent #151/#212 acceptance and installation are complete. - Confucius now owns the heavy runtime lock and frozen installation. Zeno will run - the full six-case headless journey only after an explicit serialized handoff. - No race for fresh environments, GUI, JVM, or installed-baseline changes is authorized. -- Parent also owns the cold-inspection deletion in - `InProcessCompileInspectionGateway`; Zeno does not edit that method or its - catalog-initialization call. The pending continuous regression is strengthened - to verify the secondary declaration's exact source-context relation and the - primary/secondary records' actual typed channel coordinates, in addition to - the existing exact selector and measured-secondary-region assertions. These - additional assertions are syntax-checked only until the worker slot is handed - off; no end-to-end success is inferred from source review. +## Remaining acceptance and integration boundaries + +- The original paired-channel failure was before measurement: exact channel-1 + `Nuclei` was rejected against the secondary source's channel-2 context. This + regression now **passes actual paired execution** in the unweakened six-case + journey, including declared source relation, actual typed channel coordinates, + exact selector and measured-secondary-region assertions. The 56 runtime-store + controls also pass in the current 89-case run. No source-binding owner, + source preparation, ROI, viewer or #211 authorization method/test was modified. +- The finite source-native slot is terminal and explicitly returned to the + parent. No surviving worker handle exists. Further heavy/installed acceptance + requires the next named serial handoff; no environment or blind replay is queued. +- Parent owns the cold-inspection gateway change and PR259's argument ABI and + object-label contextualization follow-up. This worker does not edit those + methods, import unpublished parent changes or use foreign installed MCP as + source acceptance. Its measurement/input-loader scope remains separate. - Source-only explanation of the apparent grouped/ungrouped distinction: group coordinates are selected by the compiled edge, while the previous predicate compared fixed coordinates on either side. Binding-owned channel identities @@ -400,8 +494,9 @@ duplicate-resolution removal is factoring; no pure file move is claimed. merge, and must not inspect Euler's blind input or output. - #204's corrected diagnosis is catalog discoverability, retaining the original failure and acceptance verbatim in `issue204_catalog_diagnosis_20260929.md`. - Same-source runtime selection is proven; paired-channel runtime and actual - GUI interaction remain open, so #204 is not claimed closed. + Exact author/compile/roundtrip, same-source and paired-channel execution are + proven here; actual installed GUI interaction remains open, so #204 is not + claimed closed. Review/merge/installation remain with the integration owner. Draft PR: https://github.com/OpenHCSDev/openhcs/pull/205. @@ -413,6 +508,7 @@ Draft PR: https://github.com/OpenHCSDev/openhcs/pull/205. - `openhcs/core/artifacts.py` - `openhcs/core/callable_contract.py` - `openhcs/core/function_patterns.py` +- `openhcs/core/pipeline/path_planner.py` - `openhcs/core/runtime_adapters.py` - `openhcs/core/component_group_scope.py` - `openhcs/core/runtime_stores.py` @@ -442,7 +538,7 @@ reproducer. Generated arrays and logs are regeneratable; no source or saved sessions were stored there. No owned large scratch artifacts were created. Viewer-bind and ROI materialization implementation surfaces remain with their assigned owners. -Current-main local build prerequisite: `a0263e82a1` adds the required +Historical local build prerequisite: `a0263e82a1` adds the required `_granularity_reconstruct` extension, reached by intensity -> shape -> zernike -> granularity imports. A fresh source import failed with `ModuleNotFoundError` before a test could execute. Zeno owns the bounded worktree-only build using the diff --git a/openhcs/core/function_patterns.py b/openhcs/core/function_patterns.py index 521851fbee..2f30a1fa2c 100644 --- a/openhcs/core/function_patterns.py +++ b/openhcs/core/function_patterns.py @@ -1222,6 +1222,7 @@ def _compile_invocation( ) item = replace(item, contract=contract_plan.contract) artifact_selector = declaration_provider(item, step_context) + item.contract.validate_artifact_input_parameter_bindings() artifact_selector.validate_artifact_output_declarations() artifact_input_plans = artifact_selector.select_plans( ArtifactInputPlan, diff --git a/openhcs/core/pipeline/path_planner.py b/openhcs/core/pipeline/path_planner.py index c5f8051cec..18b96f8f62 100644 --- a/openhcs/core/pipeline/path_planner.py +++ b/openhcs/core/pipeline/path_planner.py @@ -1250,7 +1250,6 @@ def compile_invocation_input_edges( ArtifactInputPlan, artifact_inputs, ) - invocation.contract.validate_artifact_input_parameter_bindings() input_edge_keys = InvocationArtifactInputProjectionKey.for_input_count( invocation.key, len(invocation.contract.artifact_inputs), diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index 1529af82e0..bbcf1dd880 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, replace +from csv import DictReader import numpy as np import pytest @@ -410,3 +411,11 @@ def test_headless_entrypoint_requires_subject_and_executes_corrected_rows(tmp_pa assert image.key.artifact_type is ImageArtifactType assert counts.value.data.subject.source_image_name == "CountedImage" assert tuple(counts.value.data.rows.column_values("pixel_count")) == (4,) + step_plan = bundle.runtime_contexts["A01"].step_plans[0] + [csv_path] = step_plan.artifact_analysis_output_dir.glob( + f"*_{counts.key.name}_step{step_plan.step_index}_details.csv" + ) + with csv_path.open(newline="") as stream: + [persisted] = DictReader(stream) + assert persisted["pixel_count"] == "4" + assert persisted["source_image_name"] == counts.value.data.subject.source_image_name diff --git a/tests/unit/test_artifact_output_ownership.py b/tests/unit/test_artifact_output_ownership.py index efae325793..7035136f3d 100644 --- a/tests/unit/test_artifact_output_ownership.py +++ b/tests/unit/test_artifact_output_ownership.py @@ -12,6 +12,8 @@ ImageArtifactType, ImageMeasurementSubjectRelation, MeasurementsArtifactType, + ObjectLabelsArtifactType, + ObjectMeasurementSubjectRelation, ) from openhcs.core.artifact_key_selection import ( AdapterRecordedArtifactOutputPolicy, @@ -23,7 +25,7 @@ compile_function_pattern, normalize_function_pattern, ) -from openhcs.core.pipeline.function_contracts import artifact_outputs +from openhcs.core.pipeline.function_contracts import artifact_inputs, artifact_outputs from openhcs.core.runtime_adapters import RuntimeAdapterSpec, runtime_adapter from openhcs.interop.cellprofiler.runtime.adapter import ( CellProfilerRecordedArtifactOutputPolicy, @@ -123,6 +125,43 @@ def count(image, *, runtime): compile_function_pattern(count, {}, {}) +@pytest.mark.parametrize( + "output_policy", + [ + NativeReturnArtifactOutputPolicy, + AdapterRecordedArtifactOutputPolicy, + CellProfilerRecordedArtifactOutputPolicy, + ], +) +def test_scalar_input_ambiguity_precedes_dependent_output_subject_validation( + output_policy, +): + inputs = tuple( + ArtifactSpec.input(name, ObjectLabelsArtifactType, parameter_name="labels") + for name in ("Nuclei", "Cells") + ) + rows = ArtifactSpec.output( + "Measurements", + MeasurementsArtifactType, + relations=tuple(ObjectMeasurementSubjectRelation(spec.ref()) for spec in inputs), + ) + + @runtime_adapter( + "runtime", lambda request: object(), artifact_output_policy=output_policy + ) + @artifact_inputs(*inputs) + @artifact_outputs(rows) + def measure(image, labels=None, *, runtime): + raise AssertionError("Invalid declarations must not execute") + + with pytest.raises(ValueError, match="labels.*multiple exact artifact occurrences"): + compile_function_pattern(measure, {}, {}) + + # Output obligations remain mandatory once input selection is unambiguous. + with pytest.raises(ValueError, match="multiple measurement subjects"): + CallableContract.from_callable(measure).validate_artifact_output_declarations() + + def test_cellprofiler_recording_policy_still_rejects_conflicting_explicit_subjects(): image = ArtifactSpec.output("Image", ImageArtifactType) measurements = ArtifactSpec.output( diff --git a/tests/unit/test_function_step_execution_plan.py b/tests/unit/test_function_step_execution_plan.py index 7c794bb17b..1af5495d04 100644 --- a/tests/unit/test_function_step_execution_plan.py +++ b/tests/unit/test_function_step_execution_plan.py @@ -619,7 +619,7 @@ def test_invocation_component_selection_projects_relation_owned_inputs(): manages_artifact_inputs=True, artifact_output_policy=AdapterRecordedArtifactOutputPolicy, ) - def publish_derived_channels(image, *, runtime): + def publish_derived_channels(image, *, illumination_function=None, runtime): del runtime return image diff --git a/tests/unit/test_path_planner_materialization.py b/tests/unit/test_path_planner_materialization.py index 17723ee62e..c278406f71 100644 --- a/tests/unit/test_path_planner_materialization.py +++ b/tests/unit/test_path_planner_materialization.py @@ -16,6 +16,7 @@ ArtifactSpecRelation, GroupLineageSourceRelation, ImageArtifactType, + ImageMeasurementSubjectRelation, InputGroupLineageSourceRelation, InputStackBroadcastSourceRelation, ObjectLabelsArtifactType, @@ -451,11 +452,13 @@ def identify(): "Measurements", MeasurementsArtifactType, sidecar_role=ArtifactSidecarRole.CROP_MASK, + relations=(ArtifactMeasurementSubjectRelation(),), ) image_copy_measurements = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, sidecar_role=ArtifactSidecarRole.MATERIALIZED_IMAGE_COPY, + relations=(ArtifactMeasurementSubjectRelation(),), ) base_contract = CallableContract.from_callable(identify) contracts = tuple( @@ -3839,7 +3842,10 @@ def test_site_execution_preserves_channel_grouped_producer_and_output_lineage(): measurements = ArtifactSpec.output( "Measurements", MeasurementsArtifactType, - relations=(GroupLineageSourceRelation(source.ref()),), + relations=( + GroupLineageSourceRelation(source.ref()), + ImageMeasurementSubjectRelation(source.ref()), + ), ) @artifact_inputs(source) From 9271a55f1537fca6eadb54491eb50bfa992197f4 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Wed, 30 Sep 2026 00:11:20 -0400 Subject: [PATCH 13/17] Record terminal measurement journey and current main integration --- ..._204_measurement_contract_checkpoint_20260929.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 95f0e52fe4..3b9e1d43f9 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -2,8 +2,12 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. -Base: `openhcsdev/main` at `32d070c26` (merged PR255), integrated normally through -`85d8b2ae7`. The preceding checkpoint merged `589c33a12` through `684e4bbd3`, +Base: `openhcsdev/main` at `c86f562e1` (merged disjoint PR260), integrated normally +through `b7e6a2fbc` after implementation/acceptance commit `4f4332368`. All runtime +results below were obtained before that last merge against `32d070c26` (PR255), +integrated through `85d8b2ae7`; post-merge AST and diff checks pass, with no new +native trial during the parent's slot. The preceding checkpoint merged +`589c33a12` through `684e4bbd3`, and `de23449a4` through `21c15ad41` before that. The output-policy checkpoint used `283b21275`. The earlier checkpoint used `98d9b9d23`; the runtime checkpoint integrates new @@ -52,6 +56,8 @@ Actual results, in order (counts overlap; they are not an additive unique total) was deselected or weakened. The corrected subject case now also reads the actual emitted details CSV and asserts `pixel_count=4` and `source_image_name=CountedImage` against the returned subject. + Final log: `final-six-case-and-compile-regressions.log`, SHA256 + `ef97da954948958ece3713fb1bfd5940f2de6f871bdf02554cb53efddb17710a`. The diagnostic repair moves the **existing** `CallableContract.validate_artifact_input_parameter_bindings()` call from the @@ -86,6 +92,9 @@ remaining `manages_artifact_outputs` caller exists. `git diff --check` passes. The first AST command compared the moving remote main and tried to read an upstream-only new benchmark file absent from this older worktree; the check was corrected to the actual validated main32 baseline, without editing that file. +After the normal main260 merge, the same **25 changed Python files** also pass +AST parse/in-memory compilation against current `c86f562e1`; no fixture or owner +conflict occurred. This structural recheck is not a post-merge journey rerun. Build/test processes **2304493, 2304586, 2305892, 2306949, 2320857, 2321108, 2328819** and supervisors **2304492, 2305891, 2320856, 2328817** are all terminal From 5296400d052eadc4faae10a390969d48328d2fa4 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Wed, 30 Sep 2026 00:33:29 -0400 Subject: [PATCH 14/17] Keep artifact policy and input context projection on declaration owners --- openhcs/core/artifacts.py | 10 ++++++++++ openhcs/core/callable_contract.py | 20 +++++++++---------- openhcs/core/runtime_adapters.py | 17 ++++++++++++++++ .../interop/cellprofiler/runtime/adapter.py | 14 ++----------- .../cellprofiler/runtime/artifact_binding.py | 7 ++----- tests/unit/test_artifact_output_ownership.py | 7 ++++++- tests/unit/test_runtime_value_store.py | 5 +++++ 7 files changed, 52 insertions(+), 28 deletions(-) diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index 8059b9db98..efea459e82 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -1677,6 +1677,16 @@ def require_plan_type(self) -> type["ArtifactPlan"]: ) return self.plan_type + def require_measurement_feature_owner(self) -> type["RuntimeMeasurementFeatureOwner"]: + """Require the declared row owner without inferring one from consumers.""" + if self.measurement_feature_owner is None: + raise ValueError( + f"Measurement output {self.ref()!r} requires its declared " + "measurement_feature_owner. Finalize the callable contract " + "before compiling heterogeneous rows." + ) + return self.measurement_feature_owner + def ref(self) -> ArtifactSpecRef: """Return the scope-free identity for this declaration.""" return ArtifactSpecRef( diff --git a/openhcs/core/callable_contract.py b/openhcs/core/callable_contract.py index c1acc6a7ee..0323c2337f 100644 --- a/openhcs/core/callable_contract.py +++ b/openhcs/core/callable_contract.py @@ -41,6 +41,7 @@ from openhcs.core.artifact_key_selection import ( ArtifactOutputPolicy, ArtifactPlanKeySelector, + NativeReturnArtifactOutputPolicy, ) from openhcs.core.artifacts import ( ArtifactSpec, @@ -283,6 +284,14 @@ class CallableMetadata: primary_image_carrier_requirement: PrimaryImageCarrierRequirement | None = None primary_image_carrier_transition: PrimaryImageCarrierTransition | None = None + @property + def artifact_output_policy(self) -> type[ArtifactOutputPolicy]: + """Project recording ownership from this callable's adapter declaration.""" + adapter = self.runtime_adapter + if adapter is None: + return NativeReturnArtifactOutputPolicy + return adapter.artifact_output_policy + def __post_init__(self) -> None: """Normalize the generic artifact-fed callable parameter declaration.""" @@ -968,12 +977,6 @@ def artifact_specs(self) -> ArtifactSpecCollection: (*self.metadata.artifact_inputs, *self.metadata.artifact_outputs) ) - @property - def artifact_key_specs(self) -> ArtifactSpecCollection: - """Return declarations owned by this callable's effective artifact contract.""" - - return self.artifact_specs - @property def primary_input_parameter_name(self) -> str | None: """FunctionStep input payload parameter declared by callable signature.""" @@ -1153,10 +1156,7 @@ def bind_runtime_owned(parameter_name: str) -> None: @property def artifact_output_policy(self) -> type[ArtifactOutputPolicy]: """Project the output policy from the callable's adapter declaration.""" - adapter = self.runtime_adapter - if adapter is None: - return super(CallableContract, self).artifact_output_policy - return adapter.artifact_output_policy + return self.metadata.artifact_output_policy def validate_artifact_input_parameter_bindings(self) -> None: """Validate exact artifact occurrences against the normalized callable ABI.""" diff --git a/openhcs/core/runtime_adapters.py b/openhcs/core/runtime_adapters.py index 6c46d7c729..14e0bb43cc 100644 --- a/openhcs/core/runtime_adapters.py +++ b/openhcs/core/runtime_adapters.py @@ -64,6 +64,7 @@ from openhcs.core.callable_contract import CallableContract from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.steps.function_runtime import FunctionRuntimeScope + from openhcs.core.runtime_stores import RuntimeArtifactInput _F = TypeVar("_F", bound=Callable[..., Any]) @@ -259,6 +260,22 @@ def require_artifact_input_edge( ) return first + def runtime_artifact_input( + self, + edge: InvocationArtifactInputEdgePlan, + *, + backend: str, + ) -> "RuntimeArtifactInput": + """Bind an exact compiled input to this request's declared source context.""" + from openhcs.core.runtime_stores import RuntimeArtifactInput + + return RuntimeArtifactInput( + edge_plan=edge, + axis_scope=self.axis_scope, + backend=backend, + source_binding_plan=self.source_binding_plan, + ) + def source_artifact_payload(self, ref: ArtifactSpecRef) -> object: """Resolve one source-bound artifact through workspace matching and VFS.""" diff --git a/openhcs/interop/cellprofiler/runtime/adapter.py b/openhcs/interop/cellprofiler/runtime/adapter.py index 9ae407b663..5549514b43 100644 --- a/openhcs/interop/cellprofiler/runtime/adapter.py +++ b/openhcs/interop/cellprofiler/runtime/adapter.py @@ -79,12 +79,7 @@ def validate_measurement_subject(cls, spec: ArtifactSpec) -> None: # Common relation invariants are checked by the artifact kind. CP's # module owner assembles heterogeneous rows and validates their subjects # through its row policy and add_measurements at recording time. - if spec.measurement_feature_owner is None: - raise ValueError( - f"CellProfiler measurement output {spec.ref()!r} requires its " - "declared measurement_feature_owner. Finalize the module's " - "callable contract before compiling heterogeneous rows." - ) + spec.require_measurement_feature_owner() @dataclass(slots=True) @@ -153,12 +148,7 @@ def _artifact_input( f"Compiled artifact input {name!r} has type " f"{storage_plan.artifact_type.value}, not {artifact_type.value}." ) - return RuntimeArtifactInput( - edge_plan=edge_plan, - axis_scope=self.request.axis_scope, - backend=self.backend, - source_binding_plan=self.request.source_binding_plan, - ) + return self.request.runtime_artifact_input(edge_plan, backend=self.backend) def artifact_input_records( self, diff --git a/openhcs/interop/cellprofiler/runtime/artifact_binding.py b/openhcs/interop/cellprofiler/runtime/artifact_binding.py index e1926ad220..0fbfc14e41 100644 --- a/openhcs/interop/cellprofiler/runtime/artifact_binding.py +++ b/openhcs/interop/cellprofiler/runtime/artifact_binding.py @@ -56,7 +56,6 @@ RuntimeSliceAlignedValues, ) from openhcs.core.runtime_slice_projection import RuntimeSliceProjection -from openhcs.core.runtime_stores import RuntimeArtifactInput from openhcs.core.source_matching import SourceImageSetIdentityPolicy from openhcs.core.source_plane_alignment import ( SourcePayloadPlaneIdentitySequence, @@ -654,11 +653,9 @@ def artifact_request( self.adapter.request.source_artifact_payload(spec.ref()), ) elif runtime_edge is not None: - runtime_input = RuntimeArtifactInput( - edge_plan=runtime_edge, - axis_scope=self.adapter.request.axis_scope, + runtime_input = self.adapter.request.runtime_artifact_input( + runtime_edge, backend=self.adapter.backend, - source_binding_plan=self.adapter.request.source_binding_plan, ) value = cast( RuntimeCallableArgument, diff --git a/tests/unit/test_artifact_output_ownership.py b/tests/unit/test_artifact_output_ownership.py index 7035136f3d..48b69abc57 100644 --- a/tests/unit/test_artifact_output_ownership.py +++ b/tests/unit/test_artifact_output_ownership.py @@ -94,8 +94,11 @@ def publish(image, *, runtime): return publish def validate(func): + contract = CallableContract.from_callable(func) + assert contract.metadata.artifact_output_policy is output_policy + assert contract.artifact_output_policy is output_policy if boundary == "contract": - CallableContract.from_callable(func).validate_artifact_output_declarations() + contract.validate_artifact_output_declarations() else: compile_function_pattern(func, {}, {}) @@ -273,10 +276,12 @@ def test_real_cellprofiler_declaration_compiles_without_a_table_wide_subject(): authored = next(normalize_function_pattern(step.func).iter_items()) contract = provider.plans[(0, authored.key)].contract assert contract.artifact_output_policy is CellProfilerRecordedArtifactOutputPolicy + assert contract.metadata.artifact_output_policy is CellProfilerRecordedArtifactOutputPolicy (measurement,) = contract.artifact_outputs.of_artifact_type( MeasurementsArtifactType ) assert measurement.measurement_feature_owner is IdentifyPrimaryObjectsModule + assert measurement.require_measurement_feature_owner() is IdentifyPrimaryObjectsModule assert ArtifactSpecRelation.measurement_subject_for_output(measurement) is None compiled = compile_function_pattern( diff --git a/tests/unit/test_runtime_value_store.py b/tests/unit/test_runtime_value_store.py index 50e6dbd111..f10f3af1ee 100644 --- a/tests/unit/test_runtime_value_store.py +++ b/tests/unit/test_runtime_value_store.py @@ -1475,6 +1475,11 @@ def consume_labels(image, labels): axis_scope=runtime_input.axis_scope, ) + projected = adapter.request.runtime_artifact_input(edge, backend=adapter.backend) + assert projected.edge_plan is edge + assert projected.axis_scope is adapter.request.axis_scope + assert projected.source_binding_plan is runtime_input.source_binding_plan + assert projected.records(store) == (record,) assert adapter.artifact_input_records("Nuclei", ObjectLabelsArtifactType) == (record,) request = RuntimeInputBindingRequest( adapter=adapter, kwargs={}, current_image=np.zeros((2, 2)) From 20bf7e401e4286bdabf38767910448b3d5f95dbb Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Wed, 30 Sep 2026 00:34:56 -0400 Subject: [PATCH 15/17] Use nominal postponed feature-owner return annotation --- openhcs/core/artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openhcs/core/artifacts.py b/openhcs/core/artifacts.py index efea459e82..54127d7a5e 100644 --- a/openhcs/core/artifacts.py +++ b/openhcs/core/artifacts.py @@ -1677,7 +1677,7 @@ def require_plan_type(self) -> type["ArtifactPlan"]: ) return self.plan_type - def require_measurement_feature_owner(self) -> type["RuntimeMeasurementFeatureOwner"]: + def require_measurement_feature_owner(self) -> type[RuntimeMeasurementFeatureOwner]: """Require the declared row owner without inferring one from consumers.""" if self.measurement_feature_owner is None: raise ValueError( From 04d1be413a7e2faaf92291cb88d31b3e290eefbd Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Wed, 30 Sep 2026 00:40:12 -0400 Subject: [PATCH 16/17] Record current-main journey and packaged ratchet owner cutover --- ...easurement_contract_checkpoint_20260929.md | 107 +++++++++++++++++- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 3b9e1d43f9..9780fec98e 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -3,10 +3,10 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. Base: `openhcsdev/main` at `c86f562e1` (merged disjoint PR260), integrated normally -through `b7e6a2fbc` after implementation/acceptance commit `4f4332368`. All runtime -results below were obtained before that last merge against `32d070c26` (PR255), -integrated through `85d8b2ae7`; post-merge AST and diff checks pass, with no new -native trial during the parent's slot. The preceding checkpoint merged +through `b7e6a2fbc` after implementation/acceptance commit `4f4332368`. The initial +278-test runtime checkpoint used `32d070c26` (PR255), integrated through +`85d8b2ae7`; the refreshed journey below also verifies current main260 and the +latest owner cutover. The preceding checkpoint merged `589c33a12` through `684e4bbd3`, and `de23449a4` through `21c15ad41` before that. The output-policy checkpoint used `283b21275`. @@ -18,7 +18,104 @@ Closes #133. References #204; paired-channel headless execution now passes, but actual GUI/installed acceptance remains open. This is a source-headless verified draft, not an installed/live-readiness claim. -## Current-main finite journey: terminal, working checkpoint +## Latest checkpoint: main260 journey and R0 owner correction + +Read the complete new `docs/refactor` rules/index/R0/owner override package in +Lovelace's existing PR263 worktree and the binding expanded-goal receipt. Existing +PR205 keeps its scope. No R0/L0/S1-S8 implementation, parent259 method, issue264 +export owner, managed skill or installed source was edited here. Hosted CI is not +a waiting gate; actual local behavior and enforced integration rules remain. + +At published source `9271a55f1` (main260 already normally merged), fresh locked +admission measured **15.20 GiB RAM / 30.84 GiB disk**, with only authorized +historical swap warning. All nine package roots and both native extensions +resolved inside this worktree. Subject variants: **2 passed, 4 deselected**, +3.60 seconds. The full six-case journey plus runtime/catalog/output-policy and +module-local CellProfiler controls: **108 passed**, 40.30 seconds. The ten added +backend controls cover all object-intensity features/row axes, secondary distance +tie-breaking and replacement-primary topology/relationships. This is focused +local parity evidence, **not the full native CellProfiler reference corpus**. +No external dataset/reference acquisition, download or benchmark replay occurred. + +The newly applied packaged ratchet exposed a real gap in the earlier focused +structural claim. Original comparison of main260 to `9271a55f1` returned **1**: +one foreign absence probe in the CP policy, and class-excess growth of eight lines +in `CallableContract`, one in `CellProfilerRuntimeAdapter`, and one in +`RuntimeInputBindingRequest`. The original report is retained; it is not presented +as a passing check or waived by the earlier index's historical debt total. + +Owner correction `5296400d0`: + +- `ArtifactSpec.require_measurement_feature_owner()` owns its required row-owner + obligation; CP policy calls it, rather than probing another declaration's field. + Common kind invariants and legitimate heterogeneous CP recording remain intact. +- `CallableMetadata` owns policy projection from its existing adapter declaration; + `CallableContract` projects that result. Its redundant `artifact_key_specs` + override is deleted, using the existing inherited selection unchanged. +- `RuntimeAdapterRequest.runtime_artifact_input()` supplies its own exact compiled + scope/source context. Both CP consumers call it; both duplicated constructors + are deleted. The native loader retains its existing non-adapter construction. + No matching roster, alternate store, channel-ignore flag or output exemption + was added; input management and output recording are still independent. + +These are IDEN-1/7, BOUND-2, IMPL-4/5/12 and TIME-1 owner/factoring corrections, +not blank-line/class relocation to evade a size check. Source controls after the +cutover: **159 passed**, 7.86 seconds. Strengthened metadata/policy and exact +request-context identity assertions: **79 passed**, 2.09 seconds. Counts overlap. +A subsequent ratchet report found only the redundant quoted postponed return +annotation counted as a string subscript; `20bf7e401` uses the nominal postponed +annotation directly, without any change to the tool or its rules. + +Final unweakened six-case journey at **`20bf7e401`**: **6 passed**, 7.16 seconds, +test process wall time 12.57 seconds. Admission was **16.08 GiB RAM / 30.95 GiB +disk**, under nonblocking `validation.lock`, without a new environment or GUI. +Missing subject fails before callable entry; its subject-only correction executes +and persists CSV. Exact two-producer selector, both source-document roundtrips, +same-source/paired execution, actual typed channel scopes and omitted-selector +fail-closed/single-producer controls all remain enabled. No xfail or deselection. + +Packaged ratchet at `20bf7e401` against `c86f562e1`: **exit 0**, 16 changed +production files, **zero positive deltas**. It ran the existing package directly +from `/home/ts/wt/comms-owner-startup-sol-20260929/src`, tooling revision +`dbd0965d06a8d85310c8b34032ac0baf3421c3dd`, using the required interpreter; +no install, copied script or tool edit. Coverage includes the available type- +identity, long-chain/term, foreign-probe, codec, string-subscript and class-excess +measures. This package revision does not include dispatch-arm measures; focused +source/AST/new-kind caller evidence covers this change, **not a complete NRA R1 +or global semantic proof**. R0 automation/activation remains with Lovelace. + +All supervisors/tests **2375676, 2375692, 2375870, 2390917, 2390918, 2398808, +2398809, 2403863, 2403884** are terminal and absent. No worktree process survives. +The finite native slot is explicitly returned to the parent. Actual installed +MCP/GUI verification still belongs to the integration owner; this source-native +journey does not claim installation, activation, merge or biological acceptance. + +Owned receipt/scratch root: +`/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929/pr205-post-main260-20260930`. +Logs/reports are retained, including original ratchet failures and final success. +Verified regeneratable cache/tiny synthetic pytest outputs were removed only after +all processes terminated. Final evidence SHA256: + +- `journey-runtime-catalog-parity.log`: + `071260ee6ba492f7a7f61d742c462c8fd89dd5edfd07e3027505b38c773d43b9`. +- `owner-cutover-six-case-journey.log`: + `cb171090de4e4b5fc8c0c33fbb4bd73506e115eff5766bd72ebe463d308928f2`. +- `packaged-ratchet-final.log`: + `2b64440ecc0fc076fa238faf1b76c5e5ed596d6bc22435b132453b50d20f3a3b`. + +## Persisted-format declaration (R0 rule 6) + +- `PipelineDocument` source: external, unchanged serialization/schema; existing + exact selectors and subject relations roundtrip through the original authority. +- Measurement CSV/images: external, unchanged format; the corrected synthetic + declaration persists through the existing materialization owner. +- Artifact contracts, policies and runtime input projections: internal, + process-local state reset at fresh compile/execution; no durable migration, + compatibility reader or second format is introduced. +- Catalog DTO: declaration-derived runtime projection, not a persisted store; + the existing selector field now carries its binding-owned required name. + +## Earlier finite journey: terminal, working checkpoint The parent released the finite serial slot after its terminal PR259 attempt. Every build/test phase acquired `validation.lock` nonblocking. Raw initial From 9437c4492f386732c1231de42fdcb18af0ae4f06 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Wed, 30 Sep 2026 00:47:39 -0400 Subject: [PATCH 17/17] Record current main journey and explicit owner coordination --- ...easurement_contract_checkpoint_20260929.md | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md index 9780fec98e..923467b775 100644 --- a/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md +++ b/docs/plans/issue133_204_measurement_contract_checkpoint_20260929.md @@ -2,8 +2,9 @@ Implementation owner: Zeno. Integration owner: OpenHCS issue-batch coordinator. Worktree: `/home/ts/wt/openhcs-exact-label-selection-20260929`. -Base: `openhcsdev/main` at `c86f562e1` (merged disjoint PR260), integrated normally -through `b7e6a2fbc` after implementation/acceptance commit `4f4332368`. The initial +Base: `openhcsdev/main` at `dbf1c7a8b` (merged PR259), integrated normally through +`0e04ad6c7`. Earlier main260 `c86f562e1` was integrated through `b7e6a2fbc` after +implementation/acceptance commit `4f4332368`. The initial 278-test runtime checkpoint used `32d070c26` (PR255), integrated through `85d8b2ae7`; the refreshed journey below also verifies current main260 and the latest owner cutover. The preceding checkpoint merged @@ -18,7 +19,51 @@ Closes #133. References #204; paired-channel headless execution now passes, but actual GUI/installed acceptance remains open. This is a source-headless verified draft, not an installed/live-readiness claim. -## Latest checkpoint: main260 journey and R0 owner correction +## Latest integration checkpoint: current main259 verified + +Normally merged `dbf1c7a8b` in the existing worktree at **`0e04ad6c7`** without +conflicts. Parent main/raw argument hooks and object-label strategy leaves are +unchanged by this worker. No source edit or competing fix for issue257/264, R0, +startup, source binding or the broader archive was introduced. + +Explicit shared-file flags were delivered directly on the existing owner PRs: +[Socrates PR262](https://github.com/OpenHCSDev/openhcs/pull/262#issuecomment-5904213847) +and [Lovelace PR263](https://github.com/OpenHCSDev/openhcs/pull/263#issuecomment-5904214041). +The available comms route rejected two attempted sends because sender Zeno is not +registered there; no fake participant was registered and no delivery was claimed. +The separate broader refactoring agent identity remains unconfirmed; Lovelace +is the known R0 owner, not an inferred owner of every archive surface. + +Under the explicitly retained finite native slot and nonblocking lock, fresh +admission measured **16.08 GiB available RAM / 30.57 GiB disk**, historical-swap +warning only. The worktree import path and exact merged source were verified. +`pytest -q tests/integration/test_measurement_declaration_journey.py +tests/unit/test_artifact_output_ownership.py tests/unit/test_runtime_value_store.py +--tb=short` passed **85 cases**, 37.74 seconds (43.61 seconds process wall time): +all six real declaration/compile/orchestrator/CSV journeys, all 23 output-policy +cases and all 56 runtime-input controls. No deselection, weakened assertion, +mocked replacement entrypoint, restart or replay of any original blind run. + +The existing packaged ratchet against **`dbf1c7a8b` → `0e04ad6c7`** again exits +**0**, 16 changed production files, zero positive deltas. All 25 changed Python +files parse/compile in memory; diff check passes. Coverage limits from the R0 +receipt below still apply; this is not a complete NRA R1 proof or full reference +corpus/installed GUI acceptance. The internal policy and source-context owners +still reject invalid inputs independently of output recording ownership. + +Supervisor **2418566** and pytest **2418569** exited zero and are absent; no +surviving worker runtime/test handle remains. The finite native slot is explicitly +returned to the parent. Worker-owned self-generated fixture/cache scratch was +removed after termination, retaining command and ratchet reports at +`/home/ts/.cache/agent-scratch/openhcs-issue-measurement-20260929/pr205-main259-dependency-20260930`. +Final SHA256: + +- `journey-policy-input-controls.log`: + `a0cc302bb562e24b3f109fc4dea88d2ab6d2e3e7c6abb2921a57d373038a10e0`. +- `packaged-ratchet.log`: + `50a31a25fd1f09503c5e3448d0b3bed421ac05f189c79ac48a6a4457cb9c9d8b`. + +## Main260 journey and R0 owner correction Read the complete new `docs/refactor` rules/index/R0/owner override package in Lovelace's existing PR263 worktree and the binding expanded-goal receipt. Existing