From 49c48ea6f7e80934f93312b2e236976abdfd0520 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 17:15:17 -0400 Subject: [PATCH 1/2] Keep cold plate inspection responsive and inherit declared bundle cache (#224) --- openhcs/agent/capabilities.py | 13 +- openhcs/utils/environment.py | 3 + .../test_plate_inspection_cold_runtime.py | 128 ++++++++++++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 tests/unit/agent/test_plate_inspection_cold_runtime.py diff --git a/openhcs/agent/capabilities.py b/openhcs/agent/capabilities.py index bc501e7cd1..03dd746dce 100644 --- a/openhcs/agent/capabilities.py +++ b/openhcs/agent/capabilities.py @@ -2034,6 +2034,8 @@ class GenerateSyntheticPlateCapability(PlatePathCapability): class InspectPlatePathCapability(PlatePathCapability): + progress_heartbeat_seconds = 5.0 + progress_worker_thread_safe = True name = "openhcs_inspect_plate_path" cli_command = "inspect-plate" kind = CapabilityKind.TOOL @@ -2045,9 +2047,18 @@ class InspectPlatePathCapability(PlatePathCapability): "workspace-preparation advice, and structured workflow routing. " "It does not configure a running UI or make a handler override the setup " "route; use the PlateManager code document plus selected-plate init when " - "the result must remain visible in the desktop." + "the result must remain visible in the desktop. Optional Bio-Formats " + "cold preparation can download verified Fiji artifacts into its declared " + "bundle cache and start Java; progress keeps this operation observable " + "without blocking unrelated MCP reads. Plate contents remain read-only." ) service = "selected_plate" + mutating = True + side_effects = ( + "may_download_verified_fiji_runtime", + "may_write_runtime_bundle_cache", + "may_start_java_runtime", + ) data_exposure = ( "local_plate_path", "microscope_metadata", diff --git a/openhcs/utils/environment.py b/openhcs/utils/environment.py index 0ddfab6b9e..97035ffd72 100644 --- a/openhcs/utils/environment.py +++ b/openhcs/utils/environment.py @@ -48,11 +48,14 @@ def current_numba_cache_path(cls) -> Path: def child_process_environment_keys(cls) -> tuple[str, ...]: """Return mode selectors required for semantic parity in child processes.""" + from polystore.imagej_distribution import FijiArchiveDistribution + return ( cls.cpu_only_key, cls.headless_key, cls.numba_cache_key, cls.use_threading_key, + FijiArchiveDistribution.cache_root_environment_key, ) @staticmethod diff --git a/tests/unit/agent/test_plate_inspection_cold_runtime.py b/tests/unit/agent/test_plate_inspection_cold_runtime.py new file mode 100644 index 0000000000..1ec8012df8 --- /dev/null +++ b/tests/unit/agent/test_plate_inspection_cold_runtime.py @@ -0,0 +1,128 @@ +"""Inspection uses existing progress/worker and bundle-root projection owners.""" + +import asyncio +import sys +import threading +from types import SimpleNamespace + +import pytest +from polystore.imagej_distribution import FijiArchiveDistribution + +from openhcs.agent.capabilities import InspectPlatePathCapability +from openhcs.agent.dto.common import SCHEMA_VERSION +from openhcs.agent.dto.plate import PlatePathInspectionResult +from openhcs.mcp.dev_client_core import McpDevServerSpec +from openhcs.utils.environment import OpenHCSProcessEnvironment + + +def test_mcp_launch_preserves_bundle_root_separately_from_run_cache( + monkeypatch, tmp_path +): + key = FijiArchiveDistribution.cache_root_environment_key + bundle_root = str(tmp_path / "bundles") + run_cache = str(tmp_path / "run-cache") + monkeypatch.setenv(key, bundle_root) + monkeypatch.setenv("XDG_CACHE_HOME", run_cache) + assert key in OpenHCSProcessEnvironment.child_process_environment_keys() + environment = McpDevServerSpec(sys.executable).environment() + assert environment[key] == bundle_root + assert environment["XDG_CACHE_HOME"] == run_cache + + +def test_inspection_declares_progress_and_optional_runtime_side_effects(): + capability = InspectPlatePathCapability.to_spec() + assert capability.progress_heartbeat_seconds == 5.0 + assert capability.progress_worker_thread_safe + assert capability.mutating + assert not capability.read_only + assert "may_download_verified_fiji_runtime" in capability.side_effects + assert "Plate contents remain read-only" in capability.description + + +def test_real_binding_keeps_discovery_read_responsive_during_controlled_cold_io(): + from openhcs.mcp import server + + async def exercise(): + loop = asyncio.get_running_loop() + entered = asyncio.Event() + release = threading.Event() + caller_thread = threading.get_ident() + observed_threads = [] + + class ControlledColdInspection: + def inspect(self, request): + observed_threads.append(threading.get_ident()) + loop.call_soon_threadsafe(entered.set) + assert release.wait(1), "Cold fixture was not released" + return PlatePathInspectionResult( + schema_version=SCHEMA_VERSION, + plate_path=request.plate_path, + requested_microscope_type=request.microscope_type, + ) + + built = server.build_server( + SimpleNamespace(plate_inspection_service=ControlledColdInspection()) + ) + pending = asyncio.create_task( + built.call_tool( + InspectPlatePathCapability.name, {"plate_path": "/controlled/plate"} + ) + ) + try: + await asyncio.wait_for(entered.wait(), 0.5) + assert not pending.done() + result = await asyncio.wait_for( + built.call_tool("openhcs_list_capabilities", {}), 0.5 + ) + assert result is not None + assert not pending.done(), ( + "Discovery must finish while inspection is still held" + ) + finally: + release.set() + await asyncio.wait_for(pending, 1) + assert observed_threads and observed_threads[0] != caller_thread + + asyncio.run(exercise()) + + +@pytest.mark.parametrize("fails", (False, True)) +def test_existing_progress_reports_held_work_and_propagates_terminal_failure( + monkeypatch, fails +): + from openhcs.mcp import server + + monkeypatch.setattr(InspectPlatePathCapability, "progress_heartbeat_seconds", 0.005) + + async def exercise(): + heartbeat = asyncio.Event() + messages = [] + error = RuntimeError("controlled cold initialization failed") + + class RecordingContext: + request_context = object() + + async def report_progress(self, progress, total=None, message=None): + messages.append(message) + if progress: + heartbeat.set() + + async def operation(): + await heartbeat.wait() + if fails: + raise error + return "ready" + + pending = server._await_with_declared_progress( + InspectPlatePathCapability.to_spec(), RecordingContext(), operation() + ) + if fails: + with pytest.raises(RuntimeError) as caught: + await asyncio.wait_for(pending, 1) + assert caught.value is error + else: + assert await asyncio.wait_for(pending, 1) == "ready" + assert messages[0] == "Inspect plate path: started" + assert "Inspect plate path: still running" in messages + + asyncio.run(exercise()) From bf9aae4620585e98d10e469037ef98237884b4f8 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Tue, 29 Sep 2026 17:36:28 -0400 Subject: [PATCH 2/2] Pin cache-only PolyStore checkpoint and preserve native reuse evidence (#224) --- ...agej_cache_progress_checkpoint_20260929.md | 135 ++++++++++++++++++ external/PolyStore | 2 +- openhcs/utils/environment.py | 6 +- scripts/verify_existing_imagej_bundle.py | 41 ++++++ .../test_plate_inspection_cold_runtime.py | 8 +- 5 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 docs/plans/imagej_cache_progress_checkpoint_20260929.md create mode 100644 scripts/verify_existing_imagej_bundle.py diff --git a/docs/plans/imagej_cache_progress_checkpoint_20260929.md b/docs/plans/imagej_cache_progress_checkpoint_20260929.md new file mode 100644 index 0000000000..bc85ddd6ce --- /dev/null +++ b/docs/plans/imagej_cache_progress_checkpoint_20260929.md @@ -0,0 +1,135 @@ +# Issue224 cache/progress integration checkpoint + +Owner: Lovelace implementation; parent review/integration/installation/live gate. +Fresh distinct source task after lost backend handles, not Retry of an uncertain +call. Read-only verification found original remote206=`6bc71ee5c` and16=`9e205fee` +exactly matching their persistent worktree heads. Only intentional old parent +gitlink difference present. Both original branches and all receipts preserved. + +## Exact pair and scope + +Parent base current remote main `213439a04ec7e47694cb0aec2127a0e9ca2790c2` +(includes merged218). Child base current/recorded main +`f94bbbe8631a4c78f7ceaec72fb99671b58c9a18`. +Worktree `/home/ts/wt/openhcs-imagej-cache-progress-20260929`, parent branch +`fix/imagej-cache-progress-20260929`, child branch `fix/imagej-cache-root-20260929`. +Eight recorded dependencies initialized locally using the existing source as +object references; no shared environment/source change. Child source commit +`c16b8fc99a07f17e92e5fbf46f312d73cc9c46d7` is published and the exact parent gitlink. + +Original cache commits5e656f8/101a6a4 were cherry-picked without unrelated ROI, +Java probe or source selection changes. The policy extension adds first-class +download prohibition; no additional materializer/runtime owner. Broad206/16 stay +open for those remaining changes; the small pair is their authorized integration +checkpoint, not competing implementation. Update originals by normal integration +after the parent merges the small pair, never rebase/reset/force-push. + +## Bootstrap and reproducible native admission + +Before importing OpenHCS/PolyStore or launching the MCP child, set: + +```sh +export POLYSTORE_IMAGEJ_CACHE_ROOT=/home/ts/.cache/polystore/imagej +export POLYSTORE_IMAGEJ_ALLOW_DOWNLOAD=false +``` + +Keep each run's XDG_CACHE_HOME and log roots independent. The existing +OpenHCSProcessEnvironment child projection derives both keys from their PolyStore +owners; McpDevServerSpec.environment already consumes that projection. +Unset root retains existing platformdirs default. Unset download permission keeps +existing policy defaults; explicit permission accepts only true/false. +No copy/archive/download needed on this host. An absent/invalid bundle with false +fails before creating the cache root/lock/staging or deleting any directory. + +Source validation uses `/home/ts/code/projects/openhcs/.venv/bin/python`, +OPENHCS_CPU_ONLY=true, PYTHONDONTWRITEBYTECODE=1 and PYTHONPATH to this worktree, +then these recorded dependency src directories: ObjectState, PolyStore, +arraybridge, metaclass-registry, pycodify, pyqt-reactive, python-introspect, +zmqruntime. All are under this same worktree's external directory. + +With that explicit source environment, run twice (each new process) with distinct +XDG_CACHE_HOME and the two bootstrap variables above: + +```sh +timeout 15 /home/ts/code/projects/openhcs/.venv/bin/python scripts/verify_existing_imagej_bundle.py +``` + +Executed first/second XDG values were `/home/ts/.cache/agent-scratch/openhcs-cache-proof-first-20260929` +and `.../openhcs-cache-proof-second-20260929`. Neither directory existed or was +created. Both returned the same actual runtime: + +- Fiji: `/home/ts/.cache/polystore/imagej/fiji-20260718-0417-linux_x86_64-eb5d70b3ffb6/Fiji` +- Java: that Fiji root plus `java/linux64/zulu21.42.19-ca-jdk21.0.7-linux_x64` +- Source: this worktree's external/PolyStore/src/polystore/imagej_distribution.py +- download_allowed=false, new_cache_entries=0, jvm_started=false. + +This is real native bundle materialization/admission, not JVM/MCP/installed proof. +The production policy is used directly, without DownloadForbidden override. +Before/after recursive path inventory unchanged; jpype.isJVMStarted false both +before/after. No bundle contents copied/hashed/archived/downloaded. Shared bundle, +foreign caches, frozen installation and blind receipts remain untouched. + +## Executed focused tests and bounds + +Guard: RAM14.4GiB available, swap11.1GiB warning, home20.6GiB, root8.6GiB. +No heavy runtime permitted or started. Small provider-free source shard only. +Assert available RAM>=8GiB before imports. Verified OpenHCS, distribution, +ObjectState and metaclass_registry resolve to the new assigned checkpoint tree. +Import OpenHCS before pytest collection. PYTEST_DISABLE_PLUGIN_AUTOLOAD=1; +shell timeout45; pytest.main argv: + +```text +--noconftest --import-mode=importlib -p no:cacheprovider -o addopts= -q --tb=short +external/PolyStore/tests/test_imagej_cache_environment.py +external/PolyStore/tests/test_imagej_distribution.py +tests/unit/agent/test_plate_inspection_cold_runtime.py +``` + +**36 passed,2 warnings in5.14s**, driver peakRSS277712KiB (~271.2MiB), final +RAM13.964GiB. Warnings are unused asyncio configuration with plugins disabled. +14 cache/policy +17 existing distribution +5 parent launch/progress/binding cases. +The fixture-only tests use tiny local verified ZIP responses; the actual shared +bundle proof above is separate and does not use those fixtures. +Existing real in-process FastMCP binding invokes controlled blocking inspection +on the declared worker while canonical capability discovery completes. Existing +heartbeat helper covers terminal success/error without swallowing exceptions. +No real stdio process, listen socket, JVM, GUI or scientific execution in this +shard. All test children reaped; commands terminal. No shared lock taken. +Ruff passes child source/tests and parent tests/proof; both diff checks pass. + +## Essential changed-surface catalog review + +BOUND-1/2/6: FijiArchiveDistribution.cache_root_from_environment owns bundle-only +boundary decode; ImageJArchiveDownloadPolicy.from_environment owns permission. +Canonical FIJI construction projects both once. Consumer reads typed constructors, +not repeated environment maps. New root input: one existing owner decision, zero +BioFormats/viewer edits. Download permission: one policy decoder/admission method, +shared by missing-bundle preparation and archive/overlay download; no caller fork. + +MEMB-2/3: OpenHCSProcessEnvironment derives both dependency-owned key spellings, +McpDevServerSpec already consumes the projection. No independent filename roster, +runtime/default registry or XDG alias. Strict external two-value boolean decoding +is not new runtime-kind dispatch. + +IMPL-6/12/13: InspectPlatePathCapability declares heartbeat/thread-safety and +truthful optional runtime mutations. Existing generic binding owns thread offload +and progress; no tool-name switch, job/progress store, polling runner or timeout +inflation. Another suitable synchronous capability needs only its declaration, +not a copied worker. Side effects require mutating=True under existing registry +validation; the prior declaration error was fixed, not the validator weakened. + +Coverage: focused changed source/consumer trace, catalog and executed behavioral +checks, no full NRA scan/global proof. No source preparation/ROI semantic delta +in this pair and no claim of Java lifecycle/biological fidelity. + +## Remaining acceptance and original receipts + +Parent owns merge/install of the exact small pair, then resource-coordinated real +installed stdio plate inspection plus concurrent discovery/heartbeat. No JVM +until that slot; no source/skill/environment installation by this worker. Issue224 +remains open until actual affected installed journey passes. +Original206/16 combined **141 passed/26 failed37.12s** remains explicit. Candidate +cause is metadata bootstrap/import order, not proved repaired. After-ROI profile +did not run. Native warm admission and36 source cases do not erase that evidence +or claim preservation/live readiness. Broader source/ROI and real Java/CZI gates +continue in the original drafts. No uncertain calls or scientific input replayed. diff --git a/external/PolyStore b/external/PolyStore index f94bbbe863..c16b8fc99a 160000 --- a/external/PolyStore +++ b/external/PolyStore @@ -1 +1 @@ -Subproject commit f94bbbe8631a4c78f7ceaec72fb99671b58c9a18 +Subproject commit c16b8fc99a07f17e92e5fbf46f312d73cc9c46d7 diff --git a/openhcs/utils/environment.py b/openhcs/utils/environment.py index 97035ffd72..e4bb467956 100644 --- a/openhcs/utils/environment.py +++ b/openhcs/utils/environment.py @@ -48,7 +48,10 @@ def current_numba_cache_path(cls) -> Path: def child_process_environment_keys(cls) -> tuple[str, ...]: """Return mode selectors required for semantic parity in child processes.""" - from polystore.imagej_distribution import FijiArchiveDistribution + from polystore.imagej_distribution import ( + FijiArchiveDistribution, + ImageJArchiveDownloadPolicy, + ) return ( cls.cpu_only_key, @@ -56,6 +59,7 @@ def child_process_environment_keys(cls) -> tuple[str, ...]: cls.numba_cache_key, cls.use_threading_key, FijiArchiveDistribution.cache_root_environment_key, + ImageJArchiveDownloadPolicy.allow_download_environment_key, ) @staticmethod diff --git a/scripts/verify_existing_imagej_bundle.py b/scripts/verify_existing_imagej_bundle.py new file mode 100644 index 0000000000..4037ee1fd8 --- /dev/null +++ b/scripts/verify_existing_imagej_bundle.py @@ -0,0 +1,41 @@ +"""Read-only native bundle admission proof; does not initialize ImageJ or Java. + +Set POLYSTORE_IMAGEJ_CACHE_ROOT to an already materialized cache parent and +POLYSTORE_IMAGEJ_ALLOW_DOWNLOAD=false before starting this fresh process. +""" + +import json +import os +from pathlib import Path + +import jpype +import polystore.imagej_distribution as distribution_module +from polystore.imagej_runtime import FIJI_IMAGEJ_RUNTIME + + +def main() -> None: + distribution = FIJI_IMAGEJ_RUNTIME.distribution + root = distribution.cache_root + assert root is not None and root.is_dir(), "An existing bundle cache is required" + assert not distribution.download_policy.allow_download, "Downloads must be forbidden" + assert not jpype.isJVMStarted() + before = tuple(sorted(str(path.relative_to(root)) for path in root.rglob("*"))) + launch = distribution.materialize() + after = tuple(sorted(str(path.relative_to(root)) for path in root.rglob("*"))) + assert after == before, "Materialization changed the cache inventory" + assert launch.imagej_directory.is_relative_to(root) + assert not jpype.isJVMStarted() + print(json.dumps({ + "source": str(Path(distribution_module.__file__).resolve()), + "xdg_cache_home": os.environ["XDG_CACHE_HOME"], + "shared_cache_root": str(root), + "imagej_directory": str(launch.imagej_directory), + "java_home": str(launch.java_home), + "download_allowed": distribution.download_policy.allow_download, + "new_cache_entries": 0, + "jvm_started": False, + })) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/agent/test_plate_inspection_cold_runtime.py b/tests/unit/agent/test_plate_inspection_cold_runtime.py index 1ec8012df8..96e7f58c85 100644 --- a/tests/unit/agent/test_plate_inspection_cold_runtime.py +++ b/tests/unit/agent/test_plate_inspection_cold_runtime.py @@ -6,7 +6,10 @@ from types import SimpleNamespace import pytest -from polystore.imagej_distribution import FijiArchiveDistribution +from polystore.imagej_distribution import ( + FijiArchiveDistribution, + ImageJArchiveDownloadPolicy, +) from openhcs.agent.capabilities import InspectPlatePathCapability from openhcs.agent.dto.common import SCHEMA_VERSION @@ -22,10 +25,13 @@ def test_mcp_launch_preserves_bundle_root_separately_from_run_cache( bundle_root = str(tmp_path / "bundles") run_cache = str(tmp_path / "run-cache") monkeypatch.setenv(key, bundle_root) + download_key = ImageJArchiveDownloadPolicy.allow_download_environment_key + monkeypatch.setenv(download_key, "false") monkeypatch.setenv("XDG_CACHE_HOME", run_cache) assert key in OpenHCSProcessEnvironment.child_process_environment_keys() environment = McpDevServerSpec(sys.executable).environment() assert environment[key] == bundle_root + assert environment[download_key] == "false" assert environment["XDG_CACHE_HOME"] == run_cache