diff --git a/benchmark/fragmented_roi_profile.py b/benchmark/fragmented_roi_profile.py new file mode 100644 index 0000000000..3e50d56194 --- /dev/null +++ b/benchmark/fragmented_roi_profile.py @@ -0,0 +1,81 @@ +"""Bounded provider-free profile of the declared ROI extraction/ZIP owners.""" + +from __future__ import annotations + +import argparse +import cProfile +import json +import pstats +from collections import Counter +from pathlib import Path +from time import perf_counter +from zipfile import ZipFile + +import numpy as np +from polystore.disk import DiskStorageBackend +from polystore.roi import extract_rois_from_labeled_mask +from polystore.roi_converters import FijiROIConverter, NapariShapeMetadata + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "output_dir", type=Path, help="Owned disposable scratch directory." + ) + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + labels = np.zeros((128, 128), dtype=np.int32) + labels[4:124:4, 4:124:4] = 188 + labels[1:3, 1:3] = 7 + labels[125:127, 125:127] = 42 + start = perf_counter() + extract_rois_from_labeled_mask(labels, min_area=1) + cold_extraction_seconds = perf_counter() - start + profiler = cProfile.Profile() + start = perf_counter() + rois = profiler.runcall(extract_rois_from_labeled_mask, labels, min_area=1) + warm_extraction_seconds = perf_counter() - start + stats = pstats.Stats(profiler) + bbox_scan_calls = sum( + value[1] for key, value in stats.stats.items() if key[2] == "find_objects" + ) + start = perf_counter() + members = FijiROIConverter.rois_to_imagej_members(rois) + encoding_seconds = perf_counter() - start + archive = args.output_dir / "fragmented.roi.zip" + start = perf_counter() + DiskStorageBackend().save(rois, archive) + archive_seconds = perf_counter() - start + with ZipFile(archive) as zipped: + member_count = sum(name.endswith(".roi") for name in zipped.namelist()) + print( + json.dumps( + { + "shape": list(labels.shape), + "parent_labels": [ + NapariShapeMetadata.from_metadata(roi.metadata).label + for roi in rois + ], + "fragments_by_parent": dict( + Counter( + NapariShapeMetadata.from_metadata(member.metadata).label + for member in members + ) + ), + "zip_roi_members": member_count, + "cold_extraction_seconds": cold_extraction_seconds, + "warm_extraction_seconds": warm_extraction_seconds, + "bbox_scan_calls": bbox_scan_calls, + "member_encoding_seconds": encoding_seconds, + "archive_encoding_and_write_seconds": archive_seconds, + }, + sort_keys=True, + ) + ) + # The archive writer includes its own required conversion; this does not + # imply production converts twice. The standalone conversion isolates cost. + stats.strip_dirs().sort_stats("cumulative").print_stats(15) + + +if __name__ == "__main__": + main() diff --git a/docs/plans/h002_query_preparation_diagnosis_20260929.md b/docs/plans/h002_query_preparation_diagnosis_20260929.md new file mode 100644 index 0000000000..7ac0caa666 --- /dev/null +++ b/docs/plans/h002_query_preparation_diagnosis_20260929.md @@ -0,0 +1,112 @@ +# H002 attempt4 query preparation: timing correction and bounded diagnosis + +Owner: Lovelace, original206/PolyStore16 source continuation only. +No edits to installed imagej-cache-progress source, H002 driver/receipts, +scientific pipeline or data. No repeated query, raw pixels/notebooks/biological +outputs, new JVM/runtime/download, install, profiling attach or process restart. + +## Evidence and exact timing meaning + +Named receipt: +`/home/ts/wt/openhcs-h002-uncoached-20260929/output/runtime/h002-live-attempt4/receipts/013-openhcs_query_plate_files.json`. +It records query with limit1, include_previews=false, terminal success. +No inspection of the underlying container was performed by this worker. + +Actual driver source `output/mcp_session_attempt4.py:94` sets +`started = time.monotonic()` once before stdio-client/session initialization. +Lines113-115 calculate `elapsed_seconds = time.monotonic() - started` BEFORE +awaiting session.call_tool. Therefore90.16998135202448 is session age at request +admission, NOT query duration. Original90s claim is withdrawn; no new90s query +regression established and no fix justified by that number. + +Receipt admission at1790718869.0149193, UTC2026-09-29T21:54:29.014919. +Result-bearing receipt mtime1790718875.589610, UTC21:54:35.589610. +Difference **6.574690818786621s**: admission-to-terminal-receipt-write interval, +including dispatch/serialization/receipt IO; not an exact native stage timer. +The driver rewrites the same receipt after the successful await. The parent's +next-receipt96.745 estimate is a separate upper bound including author time. +The previous cache acceptance explicitly measured0.211s differently; neither +figure proves identical fixture/warm/cold cost or a general latency reduction. + +Both driver1614516 and MCP1614529 were alive when checked, started17:52:57/58 +local time. The MCP stderr fd points at this attempt's mcp-stderr.log. +Read-only whitelist of actual MCP environ confirms: +`POLYSTORE_IMAGEJ_CACHE_ROOT=/home/ts/.cache/polystore/imagej` and +`POLYSTORE_IMAGEJ_ALLOW_DOWNLOAD=false`. +No credentials/unrelated environment values printed or retained. + +Startup stderr reports BigStitcher override from the existing digest-addressed +shared Fiji jars, then OMETiffReader initialization. These lines contain **no +timestamps**. Several reader-open lines accumulated across this continuing +session; they cannot be attributed/count-budgeted to query013 alone. File mtime +is the most recent write, not each line's event time. Do not assign a precise +JVM/decoder split from their order or repeated filenames. + +## Source-owned preparation boundary + +QueryPlateFilesCapability -> PlateInspectionService.query_files/_query_files +(:997/:1114) -> _create_handler(:1869) -> existing registered detection/handler +owners -> BioFormatsMetadataHandler.source_dataset(:64) -> registered +SourcePlaneStoreAdapter -> BioFormatsJavaAdapter._declares_path and +_discover_container -> BioFormatsJavaContext.ensure_initialized/open_reader +-> FIJI_IMAGEJ_RUNTIME.initialize -> FijiArchiveDistribution.materialize. +Limit1 bounds returned records after inventory preparation, not container/JVM +initialization. include_previews=false prevents requested previews, not metadata +discovery. No biological behavior is assumed. + +Existing ImageJRuntimePolicy.initialize separates materialization from Java: +with a fresh JVM it first materializes the canonical bundle, then configures +bundled Java with fetch=never and invokes imagej.init. If Java is already started, +it reuses Java subject to declared Java/gateway compatibility checks. +BioFormatsJavaContext additionally imports reader/metadata/format Java classes; +open_reader builds OME metadata and invokes setId before source-plane projection. +Its process-owned context/lifecycle lock retains initialized gateway state. +No alternate Java helper/cache/decoder has been added. + +Installed false policy rejects a missing/invalid bundle before cold filesystem +materialization/network. Actual environment + successful query + stderr pointing +to the shared bundle are consistent with successful warm bundle admission, +followed by cold Java/decoder preparation. They do not time either component. +Bundle provisioning and JVM/decoder preparation are distinct;228 fixed sharing +and inspection responsiveness, not all cold latency. + +Query currently does not declare inspection's progress heartbeat/worker offload, +and its description claims no detection although _query_files can create/detect +the handler. This is a source-backed exposure/observability concern, **not a +newly proved90s regression or measured stage bottleneck**. Keep it within the +existing capability/preparation ownership for a separately justified checkpoint; +do not create another progress runner or speculate about bypassing rich metadata. + +## Reproduce the diagnosis without calling MCP + +Read this single receipt and its actual driver; calculate +`receipt.stat().st_mtime - json.loads(receipt.read_text())["at"]`. +Inspect only the actual stderr fd/log and whitelist cache/download keys from the +existing process. Never replay the request to establish timing. Exact JVM, +reader-open and source-projection costs are **unavailable retrospectively**. +Future stage profiling belongs on ImageJRuntimePolicy/BioFormatsJavaContext and +existing preparation owners, using before/after monotonic events on an authorized +small synthetic run after the live science slot; do not attach/restart H002. + +## Source integration and checks (not live fixes) + +Original parent206 was6bc71ee5c, child16 was9e205fee, remote heads matched. +Normally merged fetched OpenHCS main8b090e and PolyStore main94bf443 into ONLY +their original isolated continuation branches. Merge heads parentb2edc4503, +child50c1e49; conflict resolution retains merged17/228 download-policy and launch +projection changes rather than older cache-only declarations. No rebase/reset/ +force-push or installed-tree mutation. Main's recorded c16b8fc gitlink is retained; +the original16 worktree intentionally differs while its broader gates remain. + +Six scoped source/test files parsed; both diff checks and Ruff on dependency +distribution/cache tests and parent cold-runtime tests pass. No pytest/runtime +rerun needed to answer this diagnostic and none performed. NRA/refactor-audit +coverage is focused source/consumer trace, not a global scan or native proof. +BOUND-2/6: canonical distribution/runtime/context own config/lifetime; do not +read/replace them in callers. IMPL-12/13: keep one materializer and Java lifecycle. +The merge restores current-main behavior, not a performance fix/new owner. + +Remaining original206/16 **141 passed/26 failed** and **after-ROI profile not run** +unchanged. Fresh cold stage profile, source preservation controls and broader +Java/CZI fidelity remain pending. No performance/preservation/live claim from +syntax checks or this receipt. Parent's installed228 acceptance remains separate. diff --git a/docs/plans/input_preparation_checkpoint_20260929.md b/docs/plans/input_preparation_checkpoint_20260929.md new file mode 100644 index 0000000000..23891008a0 --- /dev/null +++ b/docs/plans/input_preparation_checkpoint_20260929.md @@ -0,0 +1,353 @@ +# Input preparation checkpoint + +Implementation owner: Lovelace. Integration owner: OpenHCS coordinator. +Branch: `fix/input-preparation-20260929`. +Worktree: `/home/ts/wt/openhcs-input-preparation-20260929`. +Issues: #172, #132 and runtime follow-through #224. + +## Bundle-only cache and responsive inspection checkpoint: #224 + +Issue https://github.com/OpenHCSDev/openhcs/issues/224, implementation owner +Lovelace; existing drafts206/16. Parent owns disk recovery/integration, no +competing code patch. Coordinator reported fresh H001/H002 inspection creating +separate roughly859 MiB Fiji bundles through per-run XDG_CACHE_HOME, blocking +cold MCP and leaving queued10-second reads uncertain before disk-guard shutdown. +These are reported runtime observations, not worker-opened biological receipts. +No replay/restart, author coaching, installation or frozen-source/skill changes. + +No old supported application projection was found: cache_root was constructor- +owned but canonical FIJI_IMAGEJ_DISTRIBUTION/FIJI_IMAGEJ_RUNTIME/BioFormats context +had no process entrypoint. Inspection lacked the existing progress declaration, +so its synchronous operation ran on the MCP event-loop thread. +Source commits: OpenHCS `101a6a4b0ec80a8ce970478399fb9527ef2f5b51`, paired +PolyStore `5e656f8c20b0949139fe1273c31390931ea7ef6d`. + +- FijiArchiveDistribution declares **POLYSTORE_IMAGEJ_CACHE_ROOT** through + cache_root_environment_key at imagej_distribution.py:357. + cache_root_from_environment:368 decodes that owner key once into existing + cache_root when the canonical distribution is constructed at:520. Unset keeps + platformdirs policy; invalid explicit/relative settings fail, not fallback to + another per-run bundle. Release/digest/overlays, checksum, lock/staging and + runtime-policy authority remain unchanged. +- Existing OpenHCSProcessEnvironment.child_process_environment_keys at:48 derives + the dependency owner's key, not a copied spelling/default registry. Existing + McpDevServerSpec.environment at:300 preserves it through platform sanitization, + independently of per-run XDG_CACHE_HOME and log/cache policy. +- Existing InspectPlatePathCapability at:2036 declares five-second progress and + worker-thread execution. Existing MCP binding/heartbeat owns offloading and + observability; no new task/job/progress store or tool-name switch. Optional + downloads/cache writes/Java startup have truthful side effects and mutating + metadata; plate contents stay read-only. Artifact-plan/viewer/config owners + remain untouched. + +Exact new entrypoint: set POLYSTORE_IMAGEJ_CACHE_ROOT to an absolute reviewed +bundle directory **before process import/start**, keep per-run XDG_CACHE_HOME, +then use normal `python -m openhcs.mcp --surface full` or McpDevClient. This +requires the reviewed pair; the unchanged frozen installation does not support +the new projection. No real bundle was relocated. Do not configure retroactively +or restart/replay a live/uncertain process. Coordinator owns future startup. + +Executed **28 passed in5.07s / peak driver RSS267.4 MiB**:17 existing distribution +cases, six new root cases and five application projection/progress cases. +New files: external/PolyStore/tests/test_imagej_cache_environment.py, +external/PolyStore/tests/imagej_cache_process_fixture.py, +tests/unit/agent/test_plate_inspection_cold_runtime.py. +Two fresh Python processes import actual canonical distribution/runtime with +verified assigned paths. Only external host/artifact response is controlled: a +tiny local ZIP with real SHA-256, no overlays/JVM. Production verification, +lock/staging/discovery/reuse run. Independent XDG/log roots stay distinct; +verified download counts[1,0], one bundle directory, no temporary download +residue. Not a real859 MiB/Fiji/JVM performance claim. + +Actual in-process FastMCP binding/service invocation completes canonical +capability discovery while controlled inspection I/O is held on another thread. +Existing progress helper emits started/running and propagates the identical +terminal error. No MCP process/listen startup, JVM, GUI, remote download, +environment creation or paid provider. Not a continuous installed cold journey: +real stdio progress-token/queued reads, Java and live acceptance remain gates. + +Retain first10-pass/1-failure run4.91s/280.1 MiB: registry correctly rejected +effects without mutating metadata; fix declaration, not validator/assertions. +Two unused asyncio-option warnings with plugin autoload disabled. New tests, +helper and touched child source pass Ruff; six touched files parse, both diffs +pass whitespace. Whole environment-file Ruff reports untouched S110/BLE001 in +is_headless_mode; no broad clean claim. Driver uses recorded interpreter/explicit +parent and eight submodule src paths, verified imports, OpenHCS bootstrap before +pytest, disabled plugins/conftests and `-q --tb=short --noconftest +--import-mode=importlib -p no:cacheprovider -o addopts=`; shell45s bound. +Guard10.6 GiB available/historical swap11.9 GiB; explicit8 GiB admission, +final10.2 GiB. RSS is this driver, not continuous fleet peak-RAM evidence. + +Owned316 KiB tiny ZIP/cache/empty fixtures at the recorded scratch path removed +after results retained. A force-style cleanup was denied before execution; normal +non-force recursive removal of the validated exact owned path completed. No +source, saved sessions or blind data/output removed. All finite handles terminal; +no scientific validation-lock takeover. Original141-pass/26-failure receipt and +**not-run** after-change ROI profile remain open. Recorded gitlink staysf94bbbe; +validated paired adoption, normal main integration, preservation/profile and +installed/live acceptance await the next finite coordinator handoff. No closure. + +NRA/catalog receipt: BOUND-2/6 keeps cache_root/canonical distribution as config +authority; process consumers reference its declaration. MEMB-2/3: no second +environment/default registry, companion roster or cache copy/symlink. IMPL-1/4/13: +capability consumes existing generic progress/worker mechanism, not another +dispatch/job store. New root/run-cache values need configuration, not consumer +edits. Focused source/AST/contract checks only, not full NRA/native proof. Existing +ROI ownership and both previous review corrections remain intact. + +## Independent review follow-through: physical admission and Java lifecycle + +Review: https://github.com/OpenHCSDev/openhcs/pull/206#issuecomment-5897315214. +The complete comment reviewed parent `006361ff2`, recorded child `f94bbbe`, and +ROI source `8c886e7`. Both returned production findings were valid, despite the +earlier scoped review below. Lovelace picked up both; no competing owner/PR. +Current source commits: OpenHCS `32a2b27e22220b28f4a75db1977973aa61384a10`, +paired PolyStore `0aa057bc3b4241d77f5f3905e10e31efe2ee87e4`. + +1. IMPL-12 / BOUND-2: `NamedSourceBinding.physical_path_matches` at + `source_bindings.py:963` owns explicit-source resolution/equality and delegates + path clauses to `SourceSelector.path_filters_match:706`. That selector calls + the existing matcher/target-resolver families through `source_filters_match`. + Discovery (`SourceBindingsConfig.discovery_path_matches:1868`) and final + projection (`SourceBindingWorkspaceProjector.candidate_matches_binding:965`) + both invoke the same binding operation. Delete their duplicated exact-path + procedures and selector-filter loops, plus the projector's unused matcher + import. Config still owns global filters and alias union; the projector still + owns decoded metadata/components. A binding path-policy change now needs one + owner edit instead of synchronized consumer edits. No filename registry, + companion roster, raw record or metadata mirror. +2. IMPL-13 / BOUND-2: `BioFormatsJavaContext.is_single_file:158` owns the decoder + capability. Both it and existing `declares_path:153` use context-owned + `_probe_reader:168` for initialization, construction and unconditional close. + OpenHCS calls `context.is_single_file` at `bioformats_adapter.py:791`; delete + its entire `_is_single_file` helper and the fixture's imitation reader lifetime. + A compound reader remains discoverable when its entrypoint fails path filters; + decoded companion provenance remains available for final selection. Source + admission policy stays OpenHCS-owned. No setId/OME metadata open in the probe. + The existing valid ROI ownership/optimization is unchanged by this follow-up. + +Light executed evidence: **28 passed in 0.50s**, peak Python RSS **72.8 MiB**. +`tests/unit/test_source_binding_path_admission.py` has 16 behavior cases plus one +AST guard: relative/absolute/file-URI exact identity, directory-regex/OR-group +clauses, unknown metadata/component deferral, decoded companion filters without +replacing exact entrypoint identity, and unrestricted alias union without erasing +another alias's restrictions. `external/PolyStore/tests/test_bioformats_java.py` +has four existing context cases, six new true/false/failure/retry probe cases and +one AST guard. Controlled external Java responses exercise actual context +initialization and close, not a JVM. Guards prevent the reviewed consumer bypasses +and prevent probe operations from copying construction/close instead of using +their shared context lifetime. New tests and touched child source pass Ruff; +both repository diffs pass whitespace checks. No full NRA scan/global proof. + +Driver: recorded interpreter with explicit worktree/eight submodule src paths, +verified actual OpenHCS, SourceBindings/projector, PolyStore Java, ObjectState and +metaclass-registry imports. Import the existing `openhcs` entrypoint first, then +`pytest.main` with `-q --tb=short --noconftest --import-mode=importlib +-p no:cacheprovider -o addopts=` and the two files above. Automatic plugins +disabled; shell bound 45s. No global application fixture/viewer cleanup, MCP, +JVM, GUI, environment creation or shared validation-lock acquisition. Two pytest +warnings report unused asyncio configuration with plugins disabled. Initial +26-case run passed before adding the two ownership guards (0.23s/67.1 MiB). + +Resource guard: 11.4 GiB available / historical swap warning 11.9 GiB; explicit +8 GiB admission checked. Final available RAM 11.2 GiB; not a continuous peak-RAM +claim. Confucius's runtime/lock/frozen installation were not touched. Tiny scratch +owner Lovelace, purpose empty source fixtures, path +`/home/ts/.cache/agent-scratch/openhcs-issue-input-20260929`: 92 KiB removed after +retaining results. No source, biological input/output or saved session removed. + +These 28 contract/guard passes **do not resolve** the original combined-suite +**141 passed / 26 failed** result. Separate-context application preservation, +the retained sixteen-container/AUTO/real-TIFF controls at this new source pair, +the after-change bounded ROI profile, real Java capability/CZI fidelity and +installed user-path acceptance remain open for a coordinator-handed-off finite +slot. The profile did not run. The recorded gitlink remains `f94bbbe`; the new +OpenHCS call requires the paired context commit before integration. Coordinator +owns validated gitlink adoption, normal main integration, merge/install and live +acceptance. No issue closure or installed readiness is claimed. + +## First checkpoint: #172 + +Zero pixel overlap uses unrestricted cell positions, not empty random ranges. +Zero stage error uses exact grid positions, without consuming jitter draws. +Both layouts share the existing generator's tile-position behavior. Positive +jitter retains the original MT19937 sequence. No parameters are replaced with +nonzero defaults. + +Focused generator regressions cover ImageXpress and OperaPhenix, native and +vendor metadata, 1x1 and 2x2 grids, requested counts, deterministic pixels and +subpixel overlap. The existing randomness controls are retained. + +The continuous MCP regression uses the current real stdio entrypoint, generates +the issue's 1x1/64px/one-channel/one-Z/four-cell/A01/seed7 plate, inspects it and +samples pixels. It does not replace the transport, services or workspace owners +with mocks. It does not use a GUI, biological input or a paid provider. + +## Second checkpoint: #132 + +`SourceBindingsConfig.discovery_path_matches` applies the declared global +filters and union of binding-local path selections before unrelated store opens. +The existing filter matcher/target-resolver families still own clause behavior. +An unrestricted alias retains discovery; metadata/component-only selectors are +not guessed from filenames. Explicit source URIs use their existing resolver. +Image-file and NGFF adapters use this declaration directly. The Java adapter +only prunes a mismatching entrypoint when Bio-Formats `isSingleFile` declares it +independent. Compound entrypoints remain discoverable; the existing workspace +projector applies global filters to decoded store provenance, including companion +paths, then resolves binding/metadata/axis selections. + +The existing polymorphic `MicroscopeHandler.detect` now receives source +declarations, including during AUTO detection. No second detection interface, +filename registry, suffix-to-reader map, or companion-file roster was added. +Controlled-decoder tests use sixteen tiny physical filenames and the real +handler/orchestrator/workspace path: the explicit handler/orchestrator opens only +the selected container once; AUTO opens that selected container twice (detection +and initialization), and never opens the other fifteen. A real TIFF journey +through AUTO orchestrator initialization decodes only the selected file twice, +persists its exact reference, then loads its actual pixels. It has no substituted +decoder, JVM or orchestration implementation. + +The 128x128 fragmented fixture has three parent labels (7, 42, 188), with 900 +disconnected shapes belonging to label 188. Actual ROI materialization, disk ZIP +encoding and native reopen produce 902 members while preserving parent labels, +area, centroid and translated coordinate bounds. Summaries now say parent labels, +not cells. This demonstrates fragmentation, not duplicate biology. The existing +declared output-path selection can retain an exact int32 label TIFF without +calling the unselected ROI writer; its guard now has a regression witness. + +Recorded PolyStore profile: cold extraction 0.292s, warm profiled extraction +0.0195s, standalone member conversion 0.0686s, archive conversion/write 0.129s. +The latter includes its required conversion; standalone conversion isolates a +phase and does not imply production does both. Three contour calls yield 902 +members; two `find_objects` calls confirm a repeated whole-mask bbox scan. +These are single bounded observations, not a large-mask speedup claim. + +The paired optimization belongs to PolyStore's +`TwoDimensionalLabeledMaskROIExtractor.extract`: `regionprops` already supplies +the bbox and cached region image, but extraction separately calls `find_objects` +and constructs that binary crop again. The owner authorized the in-place change; +it is published in [PolyStore draft #16](https://github.com/OpenHCSDev/PolyStore/pull/16), +commit `8c886e7`, using those public properties and deleting the replaced scan, +crop/equality construction and cast. No new extractor/cache/registry. Source +pruning and provenance are unchanged. Scope excludes coordinator #211's +execution-session/capability write-authority repair and Zeno's #204 runtime repair. + +New child regressions cover holes with nested other-parent islands, +all borders, one-pixel bridges, diagonal contact, disconnected border pixels, +zero/nonzero origins, exact contour vertices/order, parent metadata, archive +reopen, exact full-canvas masks and one bbox scan. Parent raster-only TIFF +materialization now has an additional holes/borders fixture. These new checks +passed source AST/Ruff/diff guards. At the handed-off finite validation slot, +all sixteen new child cases and fragmented materialization checks passed, +alongside four fresh real MCP journeys. The combined dependency/application +suite did NOT fully pass: 141 passed / 26 failed in 37.12s. Failed workspace +cases looked for the generic `polystore_metadata.json` after the application +wrote `openhcs_metadata.json`. Collection/bootstrap import order is the source- +supported diagnosis; separate-context preservation verification remains open. +The first attempt had 29 passes / 138 setup errors because its cleaned scratch +parent was missing. No assertions weakened. Phase profile did not run after +the failed suite, so there is no after-change timing claim. + +Both finite commands are terminal; flock released; tagged audit shows no owned +pytest/MCP handles. Entry guard: 13.4 GiB available RAM / historical swap 11.9 +GiB. No JVM/GUI, installation, merge or frozen-harness mutation. Next runtime +check depends on the coordinator's next slot; fresh author now has priority. + +The parent recorded gitlink remains `f94bbbe8631a4c78f7ceaec72fb99671b58c9a18`. +The local isolated child retains tested source `8c886e7` plus its published +finite-validation receipt for the future paired test; +the recorded gitlink will not advance before validated paired coordinator review. +Child durable receipt: `external/PolyStore/docs/roi_region_properties_checkpoint_20260929.md`. + +Fidelity limits: executed evidence checks disconnected islands, identities, +hole/border/topology contour geometry, parent metadata, archive reopen, exact +masks and int32 TIFF raster preservation. Existing polygons represent independent contours, without +a declared hole-subtraction relationship. Even if ring coordinates and parent +metadata survive ZIP reopen, that alone is not ZIP-to-raster/viewer equivalence. +MaskShape is unsupported by the existing ImageJ ROI codec. Do not invent an +equivalence algorithm. Exact masks and label TIFFs are separate evidence. 3D and +private diagnostic fidelity remain unverified. No biology/Euler output opened. + +## Earlier essential catalog review (changed surface only) + +Audited against fetched `openhcsdev/main` at `98d9b9d23` and the working #206 +checkpoint, using the owner's current archived catalog. This is source/AST and +owner tracing, not a full NRA scan, proof replay or global clean-debt claim. +The later independent review found two bypasses this receipt missed; the +follow-through above supersedes its admission/lifecycle ownership claims. + +| Pattern | Concrete owner / witness | New-case check and disposition | +| --- | --- | --- | +| IMPL-12, copied procedure | `synthetic_data.py:827`, `SyntheticMicroscopyGenerator._site_position`; both layout loops call it at 875/881. | A shared positioning change now needs one owner edit, not two copied procedures. Zero jitter/overlap and both layout families have deterministic/count tests. Duplicate layout-position procedure removed. | +| IMPL-1/2/3/4, external family dispatch | `source_matching.py:114`, `SourceFilterMatcher`; `SourceFilterTargetResolver:232`; adapter discovery uses the existing `AutoRegisterMeta` registry (`bioformats_adapter.py:556`). | Directory plus OR-group clauses and a second/unrestricted alias require declarations only, no preparation switch edits. Ordinary numeric geometry and filesystem identity checks are not kind dispatch. The speculative second detection method was deleted; existing `detect` owns extension. | +| MEMB-1/2/3, duplicate rosters | `BioFormatsJavaAdapter._is_single_file` at 828 calls the decoder capability, not a hand-maintained extension/companion list. | A compound decoder response survives a nonmatching entrypoint and selects its decoded companion. A new single-file name uses the same clauses; no new filename registry or roster. | +| BOUND-1/2/6/7, bypassing decoded/config owners | `SourceBindingsConfig` at 1833/1842 owns path decisions; `SourceBindingWorkspaceProjector.candidate_matches_binding` at 966 owns decoded selectors. | Exact-source and metadata-only tests preserve URI resolution and defer unknown metadata. No raw configuration dictionary, `getattr` default, attribute-name switch or metadata mirror was introduced. AST guards verify the declaration matcher and single detection contract. | +| IDEN-2/6, conflated count/identity | ROI writer summary at `materialization/core.py:3047`; existing PolyStore parent-label metadata and archive converter retain identities. | Label 188 remains one parent with 900 fragments after archive reopen, not 900 new parents. New parent values need no relabeling/table edit. Summary ambiguity corrected; stronger topology fidelity remains open. | + +Guard results: eleven changed source/test/profile files parse; focused AST checks +verify existing matcher calls/adapter registry, metadata-free `isSingleFile` +instead of `setId` in the prune capability, and absence of a second detection +contract. New tests/profile pass Ruff and `git diff --check` passes. Existing +whole-file debt and PolyStore repeated-work findings remain explicitly scoped. + +## Validation and resource boundary + +All eight dependency imports were verified under this worktree's recorded +submodule roots, using `/home/ts/code/projects/openhcs/.venv/bin/python` and an +explicit worktree/submodule `PYTHONPATH`. No package installation or shared +source change. Main was fetched and integrated normally before implementation. + +First checkpoint: 20 tests passed in 2.32 seconds (zero geometry, randomness, +and initial preparation selections). Second checkpoint: 115 preparation, +Bio-Formats, source-workspace, plane-store and fragmented-materialization tests +passed in 15.54 seconds; the additional real TIFF orchestrator journey passed in +2.79 seconds. These are focused source checks, not a full suite or NRA proof. + +Real current-entrypoint evidence: four continuous MCP health, orientation, +generate, inspect and sample journeys passed in 23.19 seconds. Layouts: +ImageXpress and OperaPhenix, each with vendor and native OpenHCS metadata. +Each returns one requested uint16 image and an 8x8 pixel sample; native +metadata records one image and the requested 1x1 grid. This uses source overlays +with verified imports, not a changed installed package or an installed acceptance +claim. New test files pass Ruff; full-file Ruff reports existing source debt. + +Guards reported 12.8-14.8 GiB available RAM with a historical swap warning +(11.5-11.7 GiB used). Only small provider-free checks were run under the batch's +nonblocking lock; busy lock meant independent source work, not polling. The last +live TIFF/profile run finished and released the lock for Euler. Per the H003 +priority checkpoint, no new test/JVM/GUI run starts until Euler releases validation. + +Scratch owner: Lovelace. Purpose: tiny generated regression plates and pytest +fixtures. Path: `/home/ts/.cache/agent-scratch/openhcs-issue-input-20260929`. +Cleanup completed after preserving results here: only this owned 8.1 MiB +disposable directory was removed. Fixtures can be regenerated by the tests; +there is no retained copy of the disposable output. +The handed-off finite check regenerated 4.2 MiB of synthetic fixtures; only +that disposable directory was removed after retaining the results above. +Source, PR body and evidence stay in the persistent worktree. + +No merge/install performed. Installed acceptance is reserved to the coordinator. +Do not close either issue solely from source tests. + +## Changed paths + +Generation: `openhcs/demo/synthetic_data.py`. + +Preparation: `openhcs/core/source_bindings.py`, +`openhcs/core/source_binding_workspace.py`, `openhcs/microscopes/bioformats.py`, +`openhcs/microscopes/bioformats_adapter.py`, +`openhcs/microscopes/microscope_base.py`, +`openhcs/microscopes/source_bindings_handler.py`. + +ROI summary: `openhcs/processing/materialization/core.py`. + +Regressions: `tests/unit/test_synthetic_generator_zero_geometry.py`, +`tests/unit/agent/test_synthetic_plate_zero_geometry_journey.py`, +`tests/unit/test_bioformats_preparation_selection.py`, +`tests/unit/test_fragmented_roi_materialization.py`. + +Profile/evidence: `benchmark/fragmented_roi_profile.py`, this checkpoint document. +Paired continuation: `external/PolyStore/src/polystore/roi.py`, +`external/PolyStore/tests/test_roi_region_properties.py`, +`external/PolyStore/docs/roi_region_properties_checkpoint_20260929.md`. +Parent recorded gitlink is preserved. No reserved viewer/measurement/#211 files changed. diff --git a/docs/validation/input_preparation_installed_20261001.rst b/docs/validation/input_preparation_installed_20261001.rst new file mode 100644 index 0000000000..fd668663d0 --- /dev/null +++ b/docs/validation/input_preparation_installed_20261001.rst @@ -0,0 +1,92 @@ +Paired namespace integration and installed MCP checkpoint +========================================================= + +Parent integration owner; OpenHCS PR206 production 5ce9e3087 and paired +PolyStore PR16 production 91fd7e854. This receipt supersedes the pending +namespace/installed statements in the earlier quality receipt, not its history. +Normally merged namespace worker PR328/20 into these existing feature bases. + +Ownership and controls +---------------------- + +The existing frozen PolyStore MetadataConfig owns path/managed-path behavior. +OpenHCSMetadataConfig inherits that contract and declares the application +filename. VirtualWorkspaceBackend carries its exact config through the existing +FileManager native serialization. Microscope registration explicitly supplies +the application declaration. No dependency singleton/environment mutation, +companion metadata copy, alternate reader or duplicate registry is introduced. +BOUND-2/IMPL-12: reuse the existing owner and remove copied shared path methods. +The earlier MI metadata-detection capability and declared registry remain intact. + +Twenty-nine current application checks pass in both import orders (12.03s and +11.28s pytest), not fifty-eight distinct tests. Five namespace checks additionally +pass with independent custom environment filenames (4.04s). The unchanged +normalized BioFormats workspace test that failed without the registration binding +now passes. Actual FileManager pickle handoff is separately tested by the worker; +this does not establish a ZMQ or Java handoff. Original failures remain retained. +The first combined parent collection was killed at 520.27 MiB because optional +pytest plugins were not disabled; corrected source-qualified bounded shards use +the original provider-free plugin policy, with no input/assertion weakening. + +The original packaged structural guard at agent-comms 3b03785f passes on 5ce9e3087 +against main74eac059 for openhcs, benchmark and scripts: zero positive deltas. +MicroscopeHandler excess -13, source projector -10, generator -30; no guard +exceptions, scope moves, documentation compression or constant substitutions. +These checks do not certify full-context NRA/R1 or all ZIP refactors. + +Installed entrypoint evidence +----------------------------- + +Built OpenHCS0.8.7 and PolyStore0.3.0 paired wheels without isolation/downloads, +then installed them in an owned42 MiB candidate environment using explicit local +wheels, no dependencies and no index. Existing third-party dependencies are +read-only backing. Exact imported package files reside in the candidate. +OpenHCS wheel SHA25696a5ec632c679921e67825b88b477e8692d233c0e80c89633511a20ef76f5040; +PolyStore SHA25601b18c001f2b04f5b18f3e08a5f9c0a21a03059b52253712634e43f160024a00. +This is an installed pair, not a fresh public-registry resolver claim. + +The real installed persistent MCP dev-client process370288, startup +1790822619.1630557, reports healthy packaged resources56, unchanged source and +no restart requirement. Its environment deliberately sets the generic dependency +filename to generic-private.json while leaving the application default independent. +Read-only reopening of the preserved ImageXpress and Opera generated workspaces +finds their OpenHCS metadata and loads exact matching bounded4x4 uint16 samples. + +The same MCP process generated a new raw ImageXpress1x1 64x64 plate, four cells, +seed7, zero overlap and stage error. Source artifact-plan inspection of a complete +PipelineDocument with zero processing steps initialized the actual virtual +workspace, registered its backend, and returned one source mapping without errors. +This is an initialization/compile probe: zero execution axes/steps is deliberate, +not a processing or biological pass. Subsequent MCP sample through the persisted +virtual path loads the same exact4x4 pixels; reopening detects OpenHCS data, +one A01 well/image, grid1x1, and no preparation requirement/errors/warnings. +No generic-private.json companion was created. No arrays were analyzed outside MCP. + +The installed raw Opera1x1 generator itself succeeds, but returns unavailable-grid +warning. Actual initialization/compile then fails OperaPhenixXmlContentError: +Could not determine grid size from XML data. It is retained as a new explicit +issue172 follow-up, assigned to Averroes, not disguised by generated OpenHCS +metadata or altered inputs. Previously persisted Opera also exposes A01 metadata +versus R01C01 filename well identity; do not claim complete Opera normalization. + +Final same-process health succeeds. Normal quit is terminal with aggregate CLI +exit2 from retained invalid-command and structured Opera errors; it is not a +server crash. Independently verified MCP/client/recorder PIDs are gone. The native +recorder preserves every original request/result and timing. Owned scratch has +no active scientific runtime/viewer. No original image, held-out data, frozen +analysis, desktop or shared installed environment was changed. + +Remaining work and evidence +--------------------------- + +Ship the locally/source/installed validated namespace and ImageXpress checkpoint +without hosted-CI waiting. Keep issue172 open for Opera and real Java/CZI/native +execution acceptance. Full R1/ZIP refactors and scientific accuracy remain open. +No publication claim: the paired PolyStore public resolver route is separate. + +Durable parent ledger: +/home/ts/wt/openhcs-issue-batch-20260929/namespace-installed-20261001/ +contains local wheels/build logs, original guard JSON, complete MCP stdin/stdout/ +timing, fixtures and initialization_probe.py. Earlier source XML/logs and failed +predecessors remain in their original ledger directories. Current installed +candidate: /home/ts/wt/openhcs-namespace-installed-20261001/.venv. diff --git a/docs/validation/input_preparation_parent_20261001.rst b/docs/validation/input_preparation_parent_20261001.rst new file mode 100644 index 0000000000..7266a6373b --- /dev/null +++ b/docs/validation/input_preparation_parent_20261001.rst @@ -0,0 +1,88 @@ +Paired input-preparation parent checkpoint +========================================= + +Parent is the integration owner of existing OpenHCS206 / PolyStore16. Original +feature worktrees are unchanged. Published parente0dd14ed3 normally integrates +current main74eac059 as e8d6d32f857254cb84087ad712d1226697afab97. Child50c1e49 +normally integrates the application's recorded PolyStoredc341739 as +cb12a2acd91061d60cb6ec65981a9f5c01e18963. Final child4939979978439a6f38145788df9aac3bce65bc70 +adds only its canonical receipt to that tested production source. This checkpoint +adopts that final child gitlink; the other seven dependencies remain unchanged. + +Tested paired production candidate is72a9607cb2e562e0ef4c73c2942dcc4fdd24f75f; +subsequent receipt updates do not change its tested production source. + +Actual source validation uses Python3.12 and the existing shared interpreter's +third-party dependencies, not its installed OpenHCS/PolyStore sources. Own exact +external worktrees are populated from existing local Git objects; no package, +interpreter, Fiji bundle or JVM download occurs. Source-built native extensions +are produced by the standard setup.py build_ext --inplace in this own tree. +No shared or frozen installation is changed. + +Forty-eight dependency tests pass in0.82s: existing ROI, new RegionProperties +geometry/work-count and actual Java-context lifetime owners with controlled +external responses. The dependency process verifies its source location and +generic polystore_metadata.json configuration before collection. + +One hundred sixty-three application tests pass in19.12s: source-binding +workspace, source-plane stores, BioFormats handler/adapter/storage/SPW/validation, +physical preparation/admission, fragmented materialization and both synthetic +generator families. The fresh process imports the real openhcs entrypoint before +pytest, verifies both source paths and openhcs_metadata.json configuration, +and uses no plugin autoload or global conftests. Two unrelated asyncio config +warnings arise because those optional plugins are disabled. No assertion, +metadata filename or application/dependency API is patched for these checks. + +Historical141-pass/26-failure output remains historical. An intentional fresh +dependency-first control reproduces the current failure: the normalized-workspace +test writes the application metadata but VirtualWorkspaceBackend._load_mapping +looks for the earlier-decoded generic filename. Actual result1 failed in3.34s; +this is not a green test or a repaired embedding contract. Issue327 tracks the +actual reproducer and parent owner. The first control's +missing pytest scratch parent yields1 setup error in3.17s and is retained as a +separate harness mistake, not evidence of the product defect. Parent owns the +namespace-boundary follow-up; no singleton mutation or second filename reader is +introduced here. These separately bootstrapped passes establish current source +preservation, not global elimination of the import-order defect. + +Existing bounded128x128 ROI profile now runs: original parent labels7/42/188, +fragment counts1/1/900,902 native ZIP members, one find_objects scan. Earlier +same-fixture profile recorded two scans. Current cold extraction0.284355s, +warm profiled extraction0.019384s, separate member conversion0.060257s, +archive conversion/write0.133487s. A single-fixture work-count reduction is +observed; no general speedup or biological quality claim is made. Polygon rings +still do not declare hole subtraction; archive reopen is not raster/viewer +equivalence. Exact label TIFF and full-mask controls remain distinct. + +BOUND-2 / IMPL-13: existing RegionProperties and BioFormatsJavaContext own +crop/reader lifetime; existing NamedSourceBinding/SourceSelector own physical +admission. Consumers reuse those nominal operations. No decoder/filename roster, +alternate extractor, mirrored provenance/configuration or type/name switch is +added by this integration. Earlier focused ownership guards remain covered by +the current211 source checks. Full NRA/R1 and the owner's complete ZIP scope are +not certified. Installed stdio/native preparation, actual Java/CZI and new +generation/inspect/sample acceptance remain separate next-slot requirements. + +Raw source receipts, XMLs, profile and original negative-control errors remain +at /home/ts/wt/openhcs-issue-batch-20260929/input-parent-20261001. All source shards +have an unchanged60-second shell bound and one numerical thread. Disposable build, +pytest and profile output is tracked under the parent's named input-parent-20261001 +scratch directory. Active independent H003f's source, skills, runtime handles, +scientific inputs/settings and protected user viewers are untouched. Neither +issue132 nor172 is closed by this source-only checkpoint. + +Unchanged original structural guard +---------------------------------- + +Actual packaged ratchet implementation SHA256e323c94d49c2b72d9524a5169f123e64b4a6e46a41035ca9fb4497e49b6ca562 +matches the original CI-pinned agent-comms tool. Against main74eac059 and the +exact paired candidate72a9607, scripts passes; OpenHCS fails with +GodClassExcess:MicroscopeHandler+10 and benchmark fails with StringSubscript+2. +Full JSONs remain in the parent ledger. These are failures, not waivers or a +global architectural pass. Parent owns their correction through existing PR206: +factor the actual detection capability and use proper parent-label ownership in +the developer profile, without moving code outside the guard to hide it, replacing +string literals with constants merely to evade measurement, or compressing +documentation to game class size. Existing feature remains a visible draft. +Installed acceptance is pending the independent blind trial's terminal release; +the incomplete structural/full-NRA review is separate from that live boundary. diff --git a/docs/validation/input_preparation_quality_20261001.rst b/docs/validation/input_preparation_quality_20261001.rst new file mode 100644 index 0000000000..9a7577c747 --- /dev/null +++ b/docs/validation/input_preparation_quality_20261001.rst @@ -0,0 +1,79 @@ +Input-preparation ownership correction +===================================== + +Parent integration owner; existing PR206, normally integrated head a3179b214, +paired PolyStore16 head493997997 unchanged. Other open PR326's runtime files +and the installed independent H001c trial are not edited. Issue327 remains +separately assigned to the source-only namespace owner, Averroes; parent will +integrate its dependency/application result and verify the installed path. + +Ownership decisions +------------------- + +MicroscopeHandler mixes viewer/workspace operations with metadata discovery. +Its PR-added detection contract grew the original GodClassExcess by10. +MetadataMicroscopeDetector now owns the existing metadata association and +unchanged shared detection operation. MicroscopeHandler composes that capability +through MI; its secondary registry continues deriving declarations through MRO. +ImageXpress, OperaPhenix and OpenHCS use their original metadata leaves. +BioFormats and declared-source handlers keep their own detection implementations. +No copied method, registry roster, decoder fallback or central dispatch is added. +IMPL-8/AGENT-4 are ownership leads, not claims of a new dispatcher defect. + +This default operation was already shared. A new metadata-backed case needed +one declaration before and still needs one, not a falsely claimed edit reduction. +The new case can now exercise metadata detection without acquiring viewer or +workspace responsibilities. The family behavior and constructor/discovery +contract are the acceptance, not shorter class documentation or a changed guard. + +The developer ROI profile read label keys directly despite the existing +NapariShapeMetadata.from_metadata projection modeling these required label, +area and centroid fields. BOUND-2: the profile now uses that richer existing +owner for actual extracted ROI and converted-member metadata. It does not +substitute expected input labels, reparse archive names, introduce a mirror, +rename the existing public projection or alter arbitrary ROI metadata contracts. +The profile's extracted records supply its required fields. No new raw record +shape is spread into a consumer. No production conversion is added to hide cost. + +Executed source and profile evidence +------------------------------------ + +Source-qualified Python3.12:125 tests pass in20.22s. Both application and paired +dependency imports are asserted to reside in this owned checkout before collection. +Tests cover microscope identity/factory, metadata owner positive/missing cases, +one independent declaration, BioFormats handler/preparation selection, source +binding workspace/admission and plane stores. Two asyncio configuration warnings +come from disabled optional plugins. All assertions and fixture inputs remain. +The first invocation named a nonexistent test file and exited4 without tests; +its original log/XML remain separately, not relabeled as a product failure. + +The existing128x128 developer entrypoint actually runs after the decoder change: +parent labels7/42/188, fragments1/1/900,902 ZIP ROI members, one find_objects scan. +Cold0.296602s, profiled warm0.021064s, member conversion0.062092s, archive +conversion/write0.134282s. This confirms observed identities/work count on one +bounded synthetic fixture, not general throughput or raster/biological parity. + +Original logs/XML/profile are retained under +/home/ts/wt/openhcs-issue-batch-20260929/input-parent-20261001/quality-*. +One numeric thread,60-second shell bounds, no JVM, Fiji/package/environment +download or global install. Owned scratch input-parent-quality-20261001 is +bounded and cleaned after terminal source checks and evidence archive. + +Original guards and remaining gates +---------------------------------- + +The unchanged original packaged structural ratchet actually passes against +main74eac059 for committed production candidate1998f8ed6: both openhcs and +benchmark have zero positive deltas. MicroscopeHandler excess is now16 below +main, rather than10 above; the profile's added StringSubscript delta is gone. +Other existing improvements retain SourceBindingWorkspaceProjector excess-10, +SyntheticMicroscopyGenerator excess-30 and workspace ForeignAbsenceProbe-1. +No exception, scope move, string-constant substitution or guard change is used. +Original failed and current successful JSONs remain separate. Scripts source +is unchanged from the earlier passing revision. An AST comparison also proves +the complete detect signature/decorator/body is unchanged from a3179b214; +that syntax comparison does not itself prove MRO/metaclass runtime semantics. +Full-context NRA/R1/ZIP completion is not certified by these focused checks. +Installed native/MCP input preparation, actual Java/CZI and persisted sample +acceptance remain required. The current scientific trial owns the isolated +runtime slot; no foreign or frozen process is changed to obtain that slot. diff --git a/docs/validation/metadata_namespace_327.rst b/docs/validation/metadata_namespace_327.rst new file mode 100644 index 0000000000..dba3083cdf --- /dev/null +++ b/docs/validation/metadata_namespace_327.rst @@ -0,0 +1,156 @@ +Issue327 source-only namespace checkpoint +======================================== + +Parent is overall integration owner. This source implementation owns only the +metadata namespace/configuration boundary. Audited OpenHCS PR206 is +a3179b214b888308133e55682e964f77c2781879; PolyStore PR16 is +4939979978439a6f38145788df9aac3bce65bc70. Dependency draft PR20 extends PR16; +this application draft stacks on PR206 and pins the reviewed child source. +No parent/foreign worktree or installation was edited. + +Diagnosis, witnesses, destination owners +--------------------------------------- + +Original openhcs/__init__.py:20 projected POLYSTORE_METADATA_FILENAME through +an ambient environment default. Existing OpenHCSMetadataConfig at +core/virtual_workspace_metadata.py:39-59 owned the actual application filename, +metadata path and managed lock paths. Existing PolyStore MetadataConfig at +metadata_writer.py:54 owned a separate frozen generic config. Its workspace +reader at virtual_workspace.py:242 used the earlier-decoded generic singleton, +not the application's already richer owner. The sole application constructor at +microscopes/microscope_base.py:469 did not supply that owner. + +BOUND-2: bind the existing richer application's typed declaration instead of +reading an unrelated ambient dependency namespace. IMPL-13: shared metadata +path/lock implementation now lives once on MetadataConfig; the application +inherits it. TIME-7: delete bootstrap projection of another consumer's defaults. +No new namespace enum, string/type dispatch, mirrored config/store, singleton +mutation, filename copy, alternate reader, compatibility alias or test switch. + +Library workspace construction, persisted reopen and native PicklableBackend +connection parameters carry the exact frozen MetadataConfig value. FileManager +retains its existing declaration-derived backend discovery and worker registry +reconstruction. Native dictionary decoding uses constructor keyword binding; +the get_connection_params declaration is the sole producer of its two fields. +Generic default and helpers share their existing immutable generic owner. +Application config declares only its filename/default factory; common keys, +timeout and path/lock behavior are inherited, not copied. + +Durable metadata paths and structured SourcePixelRef JSON remain unchanged. +Runtime handoff records are produced with the current typed config. No data +conversion, namespace negotiation, environment workaround or legacy reader. + +New-case test and source checks +------------------------------ + +A third namespace now needs one MetadataConfig declaration/value and constructor +binding, not coordination of bootstrap timing and separate consumer defaults. +The library new-case test changes filename and subdirectory-key declaration to +collections, then loads exact pixels without changing any consumer/registry. +Site guards reject a workspace global path lookup, require the retained owner +and native keyword binding, prove inherited method identity, and forbid the +application bootstrap's dependency namespace projection. No exceptions. + +Python3.12 source tests use the existing third-party packages. Exact application +and dependency import locations are asserted; seven other externals are own +persistent worktrees at the application's recorded Git objects. The child used +for final product validation isee1e5438d64cb7d37e2641c2d9b3ee8870663adf; +later child receipt commits do not change tested production source. +Own native extensions were built normally with setup.py build_ext --inplace +(3.80s, peak203.67 MiB), not copied from installed packages. + +* Dependency: eleven checks pass in4.42s, peak294.78 MiB. Includes three + unchanged FileManager/persisted-workspace/pickle controls. +* Dependency-first application declaration journey: five pass in4.56s, + peak260.05 MiB. +* Application-first declaration journey: five pass in4.56s, peak260.05 MiB. +* Independent custom application/dependency environment declarations: five pass + in4.46s, peak260.23 MiB. + +Each application journey writes real persisted metadata through the application +writer, constructs the workspace with its declaration, reopens through the real +FileManager, and launches a fresh native Python recipient of its pickle. Exact +source plane pixels, source axes, registry rebinding and bounded sample pixels / +source-shape provenance are checked. The declaration journey is not a substitute +for the still-unbound microscope registration path below. + +Original and unresolved source failures +-------------------------------------- + +The exact original dependency-first negative control is reproduced unchanged: +one failed in4.35s, peak275.77 MiB, FileNotFoundError for polystore_metadata.json +after the application wrote openhcs_metadata.json. The first invocation's +missing-source-extension collection error is retained separately as a setup +failure. After the source contract change, the unchanged microscope test still +fails (one in3.26s, peak220.86 MiB) because its forbidden parent-owned constructor +does not yet bind the application owner. This is a real remaining source +integration requirement, not merely an installed/live gate. + +Parent-only hunk, requested publicly at +https://github.com/OpenHCSDev/openhcs/issues/327#issuecomment-5923060449:: + + from openhcs.core.virtual_workspace_metadata import METADATA_CONFIG + + backend = VirtualWorkspaceBackend( + plate_root=Path(plate_path), metadata_config=METADATA_CONFIG + ) + +Parent must apply this to its own _register_virtual_workspace_backend on PR206 +and integrate the paired stack normally. I did not edit microscope_base.py, +test_microscope_handler_identity.py, benchmark/fragmented_roi_profile.py, +PR326 runtime/preparation files or any parent external directory. No original +test assertion is modified. The first authored application worker fixture +pre-imported arraybridge's installed source before activating pinned externals; +its two failures are preserved. The worker now faithfully imports dependency +metadata before the real application entrypoint, then other source owners. + +Audit coverage and proof limits +------------------------------- + +Archive entrypoint, pattern README, complete boundaries/implementation/over-time +references and both skills were read. Lightweight full-package AST census at +pinned heads completes without parse omissions: dependency12,209 code lines +in1.06s and application355,153 code lines in22.03s. These are syntax census, +not completed NRA ownership/native-equivalence proofs. + +The unchanged original structural ratchet passes for final child source against +PR16: StringSubscript-2, other deltas zero. The first child draft's+2 failure is +retained, not waived. Actual tool SHA256 is +e323c94d49c2b72d9524a5169f123e64b4a6e46a41035ca9fb4497e49b6ca562. +Pinned tool package cannot import on3.12 (NameError: InputDocument); the existing +system3.14.7, matching original CI's3.14, executes its unchanged pinned local +Git source. This does not alter product3.12 validation or install/download +anything. Tool worktree: /home/ts/wt/metadata-namespace-327-audit-tool. + +The application guard also passes unchanged at committeda1a1689f3 against PR206: +all deltas zero for openhcs (14.01s, peak74.06 MiB), scripts (2.02s) and +benchmark (2.59s). No parent-owned files are added to the write set. +Changed-file census completes on both stacks with no parse omissions: dependency +StringSubscript-2 / literal_key_get-1, application all measures unchanged. This +does not certify parent changes after the pinned baseline or a future integration. + +Semantic edits were hand-authored; no NRA transaction or completed R1/raw-record +certificate is claimed. Full NRA/R1 remains uncompleted, not waived. All raw +logs/XML/command bounds, including failures, are retained in +metadata_namespace_327_source_evidence.tgz. Initial census JSON filenames were +overwritten by the command-summary harness; the complete census stdout remains +in logs. Harness now uses distinct .command.json outputs. No science data is +part of this archive. + +Visible paired drafts: https://github.com/OpenHCSDev/PolyStore/pull/20 and +https://github.com/OpenHCSDev/openhcs/pull/328. Final paired source uses child +91fd7e854ca760a16e7472d8d558b4012e89b6ba, whose production tree equals the tested +ee1e5438d64cb7d37e2641c2d9b3ee8870663adf tree (the later change is receipt-only). + +Acceptance boundary +------------------- + +Parent must finish the forbidden registration hunk and rerun the unchanged +microscope workspace journey in both import orders. Then parent owns paired +merge/install and the actual affected installed native/MCP entrypoint, persisted +workspace reopen and exact pixel/provenance acceptance. No merge, activation, +installed-readiness or issue closure is claimed. No UI/MCP/scientific dataset, +science-runtime lock, Fiji/environment/interpreter download or installed-package +mutation occurred. Resource ownership/bounds are recorded separately in +metadata_namespace_327_resources.rst; terminal disposable scratch is archived +and cleaned by this owner. diff --git a/docs/validation/metadata_namespace_327_resources.rst b/docs/validation/metadata_namespace_327_resources.rst new file mode 100644 index 0000000000..7eabc5070d --- /dev/null +++ b/docs/validation/metadata_namespace_327_resources.rst @@ -0,0 +1,36 @@ +Issue327 source resource ledger +============================== + +Owner: SOURCE-ONLY issue327 implementation agent. Parent owns integration. +Source worktrees: /home/ts/wt/openhcs-metadata-namespace-327-20261001 and +/home/ts/wt/polystore-metadata-namespace-327-20261001. +Disposable scratch: /home/ts/.cache/agent-scratch/metadata-namespace-327. +Purpose: tiny synthetic fixtures, pytest XML/logs and bounded source evidence. +Limits: one CPU, 512 MiB combined process RSS, 256 MiB scratch, 60s per shard. +Initial resource guard: warning, 8.5 GiB home free, 16.5 GiB RAM available, +8.9 GiB historical swap used. The owner's 20 GiB disk threshold is warning-only; +the bounded scratch maximum is less than available disk. No science/runtime lock, +installed package, GUI/MCP, provider, interpreter or environment changes. +Archive terminal command output in this worktree before removing owned scratch. + +Own read-only guard-tool worktree: +/home/ts/wt/metadata-namespace-327-audit-tool, pinned3b03785f45df2ef5dc62ba6aed99294192ecbb01, +3.1 MiB sparse checkout. Existing system3.14.7 executes the unchanged original +CI tool; product/native test processes use3.12. Source native build outputs are +openhcs/core/_tabular_native.abi3.so and +openhcs/processing/backends/cellprofiler/_granularity_native.abi3.so in this own +application worktree (376,344 bytes total); never installed/copied externally. +Final scratch is approximately4 MiB, well below256 MiB. All shards are terminal; +peak observed process-group RSS355.73 MiB, including a retained initial guard +failure, and longest shard22.03s. Archive the raw errors and results, then remove +only this owned scratch, source-native build outputs and clean guard-tool WT. +Keep both implementation WTs and their seven source dependency WTs persistent. + +Cleanup completed after terminal-process check. Fresh extraction and byte-for-byte +comparison verified original negative log/XML, current unresolved-source XML and +both complete structural-guard logs before deletion. Archive SHA256: +05b24739c81fbdc86c3dbc9e96f45e1764a5f50ffe946953271c386df68c8dfa. +Owned scratch, two reproducible native build outputs and clean tool worktree are +removed; all implementation source, parent original errors and saved history +remain untouched. Rebuild the two source extensions with the recorded standard +setup.py command before rerunning application source tests in this own WT. diff --git a/docs/validation/metadata_namespace_327_source_evidence.tgz b/docs/validation/metadata_namespace_327_source_evidence.tgz new file mode 100644 index 0000000000..6c2dd1e466 Binary files /dev/null and b/docs/validation/metadata_namespace_327_source_evidence.tgz differ diff --git a/docs/validation/run_metadata_namespace_327.py b/docs/validation/run_metadata_namespace_327.py new file mode 100644 index 0000000000..c5811c1f78 --- /dev/null +++ b/docs/validation/run_metadata_namespace_327.py @@ -0,0 +1,57 @@ +"""Bound a source command's whole process group and retain original output.""" + +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +SCRATCH = Path("/home/ts/.cache/agent-scratch/metadata-namespace-327") +SCRATCH.mkdir(parents=True, exist_ok=True) +name, *command = sys.argv[1:] +start = time.monotonic() +peak = 0 +reason = None +with (SCRATCH / f"{name}.log").open("w") as output: + process = subprocess.Popen( + command, stdout=output, stderr=subprocess.STDOUT, start_new_session=True + ) + while process.poll() is None: + rss = 0 + for stat_path in Path("/proc").glob("[0-9]*/stat"): + try: + stat = stat_path.read_text().rpartition(") ")[2].split() + if int(stat[2]) == process.pid: + rss += int(stat[21]) * os.sysconf("SC_PAGE_SIZE") + except (OSError, ValueError, IndexError): + continue + peak = max(peak, rss) + scratch_bytes = sum(p.stat().st_size for p in SCRATCH.rglob("*") if p.is_file()) + if rss > 512 * 1024**2: + reason = "512 MiB combined RSS limit" + elif scratch_bytes > 256 * 1024**2: + reason = "256 MiB scratch limit" + elif time.monotonic() - start > 60: + reason = "60s shard limit" + if reason: + os.killpg(process.pid, signal.SIGKILL) + break + time.sleep(0.05) + result = process.wait() +summary = { + "command": command, + "exit": result, + "limit": reason, + "peak_rss_mib": round(peak / 1024**2, 2), + "seconds": round(time.monotonic() - start, 2), +} +(SCRATCH / f"{name}.command.json").write_text(json.dumps(summary, indent=2) + "\n") +print(json.dumps(summary)) +log_text = (SCRATCH / f"{name}.log").read_text() +if len(log_text) <= 8000: + print(log_text) +else: + print(f"Full output retained in {SCRATCH / f'{name}.log'} ({len(log_text)} characters)") +sys.exit(result if result >= 0 else 1) diff --git a/external/PolyStore b/external/PolyStore index dc3417392c..84f322e468 160000 --- a/external/PolyStore +++ b/external/PolyStore @@ -1 +1 @@ -Subproject commit dc3417392cef19083a907c8c9b88a15466f81df4 +Subproject commit 84f322e46871de5ed47e7fd20976ad03e533c1c4 diff --git a/openhcs/__init__.py b/openhcs/__init__.py index 0fc21d6fa3..bcf7df4e9a 100644 --- a/openhcs/__init__.py +++ b/openhcs/__init__.py @@ -7,7 +7,6 @@ registrations or other side-effects. """ -import os import platform import sys @@ -16,8 +15,6 @@ __version__ = "0.8.7" -# Configure polystore defaults for OpenHCS integration -os.environ.setdefault("POLYSTORE_METADATA_FILENAME", "openhcs_metadata.json") OpenHCSProcessEnvironment.project_dependency_gpu_import_policy() OpenHCSProcessEnvironment.project_numba_worker_profiling_policy() diff --git a/openhcs/core/source_binding_workspace.py b/openhcs/core/source_binding_workspace.py index 980da5b2f8..02230380d8 100644 --- a/openhcs/core/source_binding_workspace.py +++ b/openhcs/core/source_binding_workspace.py @@ -35,7 +35,6 @@ overlay_source_metadata, semantic_source_metadata_value, source_component_metadata_values, - source_filters_match, source_metadata_component, source_metadata_value, source_metadata_values_equal, @@ -678,6 +677,9 @@ def projection_set_for_candidates( ), ) for candidate in candidates + if self.source_bindings.source_path_filters_match( + candidate.source_filter_path_identities() + ) ) candidates = tuple( @@ -902,13 +904,7 @@ def source_candidates( filter_paths = tuple( dict.fromkeys((relative_path, _normalized_source_path(path))) ) - if not any( - source_filters_match( - filter_path, - self.source_bindings.source_filter_declarations, - ) - for filter_path in filter_paths - ): + if not self.source_bindings.source_path_filters_match(filter_paths): continue metadata = self.source_bindings.coerce_metadata( metadata_from_rules( @@ -973,22 +969,15 @@ def candidate_matches_binding( source_root: Path, ) -> bool: """Return whether one candidate satisfies one typed binding selector.""" - if binding.explicit_source is not None: - explicit_path = Path( - binding.explicit_source.resolved(source_root).uri - ).resolve() - candidate_path = Path(candidate.relative_path) - if not candidate_path.is_absolute(): - candidate_path = Path(source_root) / candidate_path - if candidate_path.resolve() != explicit_path: - return False + if not binding.physical_path_matches( + source_root, + candidate.relative_path, + candidate.source_filter_path_identities(), + ): + return False selector = binding.selector return ( - any( - source_filters_match(path, selector.filters) - for path in candidate.source_filter_path_identities() - ) - and all( + all( ( value := semantic_source_metadata_value( candidate.metadata, item.field diff --git a/openhcs/core/source_bindings.py b/openhcs/core/source_bindings.py index 56dead442c..7f2bf54e30 100644 --- a/openhcs/core/source_bindings.py +++ b/openhcs/core/source_bindings.py @@ -703,6 +703,12 @@ def __post_init__(self) -> None: ), ) + def path_filters_match(self, identities: tuple[str, ...]) -> bool: + """Match store-owned path provenance, without guessing decoded identity.""" + from openhcs.core.source_matching import source_filters_match + + return any(source_filters_match(path, self.filters) for path in identities) + def source_alias_measurement_names(alias: str) -> tuple[str, ...]: """Return measurement source-name tokens represented by a source alias.""" @@ -954,6 +960,26 @@ def __post_init__(self) -> None: object.__setattr__(self, "source_channel_axis", source_channel_axis) object.__setattr__(self, "source_channel_counts", source_channel_counts) + def physical_path_matches( + self, + source_root: Path, + source_path: str | Path, + path_identities: tuple[str, ...], + ) -> bool: + """Admit one physical source using exact identity and path provenance. + + Companion identities may satisfy selector filters, but do not replace + the exact source URI. Metadata/components require subsequent decoding. + """ + if self.explicit_source is not None: + explicit_path = Path(self.explicit_source.resolved(source_root).uri) + candidate_path = Path(source_path) + if not candidate_path.is_absolute(): + candidate_path = source_root / candidate_path + if candidate_path.resolve() != explicit_path.resolve(): + return False + return self.selector.path_filters_match(path_identities) + def source_channel_axis_for_shape( self, shape: tuple[int, ...], @@ -1830,6 +1856,36 @@ def source_filter_declarations(self) -> tuple[SourceFilterClause, ...]: return tuple(self.source_filters or ()) + def source_path_filters_match(self, identities: tuple[str, ...]) -> bool: + """Match the declared source universe against physical path identities.""" + from openhcs.core.source_matching import source_filters_match + + return any( + source_filters_match(identity, self.source_filter_declarations) + for identity in identities + ) + + def discovery_path_matches(self, root: Path, path: Path) -> bool: + """Apply known physical-path selections before decoding a source store. + + Metadata and component selectors still belong to plane projection. An + unrestricted binding keeps discovery unrestricted, not just the first + filtered alias. All matching uses the existing source-filter owners. + """ + root = root.resolve(strict=False) + resolved = path.resolve(strict=False) + identities = (resolved.as_posix(),) + if root.is_file(): + identities = (resolved.name, *identities) + elif resolved.is_relative_to(root): + identities = (resolved.relative_to(root).as_posix(), *identities) + if not self.source_path_filters_match(identities): + return False + return not self.binding_declarations or any( + binding.physical_path_matches(root, resolved, identities) + for binding in self.binding_declarations + ) + def declaration_identity(self) -> str: """Return a stable identity for this complete source declaration.""" diff --git a/openhcs/core/virtual_workspace_metadata.py b/openhcs/core/virtual_workspace_metadata.py index c9aabd2da3..6e8cc71b6c 100644 --- a/openhcs/core/virtual_workspace_metadata.py +++ b/openhcs/core/virtual_workspace_metadata.py @@ -5,12 +5,13 @@ import logging import os from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType from typing import Any, Callable, TypeAlias -from polystore.atomic import LOCK_CONFIG, FileLockError, atomic_update_json +from polystore.atomic import FileLockError, atomic_update_json +from polystore.metadata_writer import MetadataConfig from polystore.virtual_workspace import SourcePixelRef from openhcs.constants.constants import AllComponents @@ -36,24 +37,14 @@ @dataclass(frozen=True) -class OpenHCSMetadataConfig: +class OpenHCSMetadataConfig(MetadataConfig): """Configuration owned by the OpenHCS metadata file contract.""" - METADATA_FILENAME: str = os.getenv( - "OPENHCS_METADATA_FILENAME", - "openhcs_metadata.json", + METADATA_FILENAME: str = field( + default_factory=lambda: os.getenv( + "OPENHCS_METADATA_FILENAME", "openhcs_metadata.json" + ) ) - SUBDIRECTORIES_KEY: str = "subdirectories" - AVAILABLE_BACKENDS_KEY: str = "available_backends" - DEFAULT_TIMEOUT: float = LOCK_CONFIG.DEFAULT_TIMEOUT - - def metadata_path(self, plate_root: str | Path) -> Path: - return Path(plate_root) / self.METADATA_FILENAME - - def managed_paths(self, plate_root: str | Path) -> tuple[Path, Path]: - """Files this metadata transaction owns, not scientific source artifacts.""" - path = self.metadata_path(plate_root) - return path, LOCK_CONFIG.lock_path(path) METADATA_CONFIG = OpenHCSMetadataConfig() diff --git a/openhcs/demo/synthetic_data.py b/openhcs/demo/synthetic_data.py index c87f8b0ee1..8a277bb2ec 100644 --- a/openhcs/demo/synthetic_data.py +++ b/openhcs/demo/synthetic_data.py @@ -240,7 +240,7 @@ def __init__( # Very strongly favor overlap regions with 80% probability # to ensure very high density of features in the 10% overlap region for reliable registration - if self._random.random() < 0.8: + if overlap_x > 0 and overlap_y > 0 and self._random.random() < 0.8: # Position in an overlap region between tiles col = self._random.randint(0, grid_size[1]) row = self._random.randint(0, grid_size[0]) @@ -824,6 +824,18 @@ def _calculate_image_size( return (width, height) + def _site_position(self, row: int, col: int) -> tuple[int, int]: + """Position a tile, preserving an exact grid when stage jitter is zero.""" + x = col * self.step_x + y = row * self.step_y + if self.stage_error_px > 0: + x += self._random.randint(-self.stage_error_px, self.stage_error_px) + y += self._random.randint(-self.stage_error_px, self.stage_error_px) + return ( + max(0, min(x, self.image_size[0] - self.tile_size[0])), + max(0, min(y, self.image_size[1] - self.tile_size[1])), + ) + def generate_dataset(self): """Generate the complete dataset.""" print(f"Generating synthetic microscopy dataset in {self.output_dir}") @@ -860,55 +872,13 @@ def generate_dataset(self): # Get grid position for this field ID grid_row, grid_col = field_pattern[site_index] - # Calculate base position - x = grid_col * self.step_x - y = grid_row * self.step_y - - # Add random stage positioning error - x_error = self._random.randint( - -self.stage_error_px, self.stage_error_px - ) - y_error = self._random.randint( - -self.stage_error_px, self.stage_error_px - ) - - x_pos = x + x_error - y_pos = y + y_error - - # Ensure we don't go out of bounds - x_pos = max(0, min(x_pos, self.image_size[0] - self.tile_size[0])) - y_pos = max(0, min(y_pos, self.image_size[1] - self.tile_size[1])) - - site_positions[site_index] = (x_pos, y_pos) + site_positions[site_index] = self._site_position(grid_row, grid_col) else: # ImageXpress: simple raster pattern site_index = 1 for row in range(self.grid_size[0]): for col in range(self.grid_size[1]): - # Calculate base position - x = col * self.step_x - y = row * self.step_y - - # Add random stage positioning error - x_error = self._random.randint( - -self.stage_error_px, self.stage_error_px - ) - y_error = self._random.randint( - -self.stage_error_px, self.stage_error_px - ) - - x_pos = x + x_error - y_pos = y + y_error - - # Ensure we don't go out of bounds - x_pos = max( - 0, min(x_pos, self.image_size[0] - self.tile_size[0]) - ) - y_pos = max( - 0, min(y_pos, self.image_size[1] - self.tile_size[1]) - ) - - site_positions[site_index] = (x_pos, y_pos) + site_positions[site_index] = self._site_position(row, col) site_index += 1 # For multiple Z-stack levels diff --git a/openhcs/microscopes/bioformats.py b/openhcs/microscopes/bioformats.py index d739c298f7..2ba34a7942 100644 --- a/openhcs/microscopes/bioformats.py +++ b/openhcs/microscopes/bioformats.py @@ -52,12 +52,19 @@ def extract_component_coordinates(self, component_value: str) -> tuple[str, str] class BioFormatsMetadataHandler(MetadataHandler): """Metadata view over exact store-emitted source planes.""" - def __init__(self, filemanager: FileManager | None = None): + def __init__( + self, + filemanager: FileManager | None = None, + source_bindings: SourceBindingsConfig | None = None, + ): super().__init__() self.filemanager = filemanager + self.source_bindings = source_bindings or SourceBindingsConfig() def source_dataset(self, plate_path: Union[str, Path]) -> SourcePlaneDataset: - return SourcePlaneStoreAdapter.discover_dataset(plate_path) + return SourcePlaneStoreAdapter.discover_dataset( + plate_path, source_bindings=self.source_bindings + ) def source_diagnostics( self, @@ -157,6 +164,22 @@ class BioFormatsHandler(BroadMicroscopeDetector, MicroscopeHandler): _microscope_type = Microscope.BIOFORMATS.value _metadata_handler_class = BioFormatsMetadataHandler + @classmethod + def detect( + cls, + plate_folder: Path, + filemanager: FileManager, + source_bindings_config: SourceBindingsConfig | None = None, + ) -> bool: + """Keep source selections ahead of container opens during auto-detection.""" + try: + BioFormatsMetadataHandler( + filemanager, source_bindings_config + ).find_metadata_file(plate_folder) + except (FileNotFoundError, TypeError): + return False + return True + @classmethod def create( cls, @@ -182,11 +205,13 @@ def __init__( source_bindings_config: SourceBindingsConfig | None = None, ): self.parser = BioFormatsFilenameParser(filemanager, pattern_format) - self.source_metadata_handler = BioFormatsMetadataHandler(filemanager) - self.metadata_handler: MetadataHandler = self.source_metadata_handler self.source_bindings = source_bindings_defaults_to_base( source_bindings_config or SourceBindingsConfig() ) + self.source_metadata_handler = BioFormatsMetadataHandler( + filemanager, self.source_bindings + ) + self.metadata_handler: MetadataHandler = self.source_metadata_handler super().__init__(parser=self.parser, metadata_handler=self.metadata_handler) @property diff --git a/openhcs/microscopes/bioformats_adapter.py b/openhcs/microscopes/bioformats_adapter.py index 0554c1f85d..761723289e 100644 --- a/openhcs/microscopes/bioformats_adapter.py +++ b/openhcs/microscopes/bioformats_adapter.py @@ -26,6 +26,7 @@ from openhcs.constants.constants import AllComponents, Backend from openhcs.core.image_file_serialization import ImageFileFormat +from openhcs.core.source_bindings import SourceBindingsConfig from openhcs.core.source_matching import ( merge_source_metadata, with_source_component_metadata, @@ -487,6 +488,17 @@ class SourcePlaneStoreAdapter(ABC, metaclass=AutoRegisterMeta): __skip_if_no_key__ = True registry_key: ClassVar[str | None] = None + def __init__(self, source_bindings: SourceBindingsConfig | None = None): + self.source_bindings = source_bindings or SourceBindingsConfig() + + def selected_source_paths(self, root: Path) -> tuple[Path, ...]: + """Select physical entrypoints before opening unrelated containers.""" + return tuple( + path + for path in _candidate_source_paths(root) + if self.source_bindings.discovery_path_matches(root, path) + ) + def source_metadata_for_path(self, path: Path) -> SourceMetadataMapping: """Enrich a filename-bound physical source without replacing its axes.""" del path @@ -532,13 +544,17 @@ def retain_candidate( return True @classmethod - def discover_dataset(cls, root: str | Path) -> SourcePlaneDataset: + def discover_dataset( + cls, root: str | Path, *, source_bindings: SourceBindingsConfig | None = None + ) -> SourcePlaneDataset: root_path = Path(root).resolve(strict=False) if not root_path.exists(): raise BioFormatsAdapterUnavailableError( f"Plane-store collection does not exist: {root_path}" ) - adapters = tuple(adapter_type() for adapter_type in cls.__registry__.values()) + adapters = tuple( + adapter_type(source_bindings) for adapter_type in cls.__registry__.values() + ) collection_owners = tuple( adapter for adapter in adapters if adapter.claims_collection(root_path) ) @@ -626,6 +642,7 @@ def discover_stores(self, root: Path) -> tuple[SourcePlaneDataset, ...]: return tuple( self._discover_store(root, location) for location in OmeZarrLocation.discover(root) + if self.source_bindings.discovery_path_matches(root, Path(location.path)) ) def _discover_store( @@ -769,6 +786,11 @@ def discover_stores(self, root: Path) -> tuple[SourcePlaneDataset, ...]: continue if not self._declares_path(context, source_path): continue + if ( + not self.source_bindings.discovery_path_matches(root, source_path) + and context.is_single_file(source_path) + ): + continue try: dataset = self._discover_container(root, source_path) except BioFormatsNoScalarSourceError as exc: @@ -877,7 +899,7 @@ def retain_candidate( def discover_stores(self, root: Path) -> tuple[SourcePlaneDataset, ...]: datasets = [] identity = SourceDatasetIdentity.for_root(root) - for source_path in _candidate_source_paths(root): + for source_path in self.selected_source_paths(root): if not ImageFileFormat.is_image_path(source_path): continue image_format = ImageFileFormat.require_path(source_path) diff --git a/openhcs/microscopes/microscope_base.py b/openhcs/microscopes/microscope_base.py index 4223bec70d..24275e5779 100644 --- a/openhcs/microscopes/microscope_base.py +++ b/openhcs/microscopes/microscope_base.py @@ -137,7 +137,50 @@ def source_selection_guidance(cls) -> str: ) -class MicroscopeHandler(ViewerMicroscopeHandlerABC, ABC, metaclass=AutoRegisterMeta): +class MetadataMicroscopeDetector: + """Detect sources through their declared microscope metadata owner. + + Image-container detectors override this operation; ordinary metadata-backed + handlers share it independently of viewer transport and workspace operations. + """ + + _metadata_handler_class: Optional[Type[MetadataHandler]] = None + + @classmethod + def detect( + cls, + plate_folder: Path, + filemanager: FileManager, + source_bindings_config: Optional["SourceBindingsConfig"] = None, + ) -> bool: + """Detect this handler; store-backed declarations can narrow discovery. + + Ordinary metadata handlers detect one plate declaration, without opening + its image containers. Store-backed handlers own selected discovery in + their implementation of this same method. + """ + from polystore.exceptions import MetadataNotFoundError + + metadata_handler_class = cls._metadata_handler_class + if metadata_handler_class is None: + raise RuntimeError( + f"{cls.__name__} missing _metadata_handler_class for detection" + ) + + metadata_handler = metadata_handler_class(filemanager) + try: + metadata_handler.find_metadata_file(plate_folder) + except (MetadataNotFoundError, FileNotFoundError, TypeError): + return False + return True + + +class MicroscopeHandler( + MetadataMicroscopeDetector, + ViewerMicroscopeHandlerABC, + ABC, + metaclass=AutoRegisterMeta, +): """ Composed class for handling microscope-specific functionality. @@ -160,9 +203,6 @@ class MicroscopeHandler(ViewerMicroscopeHandlerABC, ABC, metaclass=AutoRegisterM DEFAULT_MICROSCOPE = "auto" _handlers_cache = None - # Optional class attribute for explicit metadata handler registration - _metadata_handler_class: Optional[Type[MetadataHandler]] = None - def __init__( self, parser: Optional[FilenameParser], metadata_handler: MetadataHandler ): @@ -236,24 +276,6 @@ def source_selection_guidance(cls) -> str: "explicitly and expect metadata-derived fields to remain unavailable." ) - @classmethod - def detect(cls, plate_folder: Path, filemanager: FileManager) -> bool: - """Return whether this handler can initialize the plate.""" - from polystore.exceptions import MetadataNotFoundError - - metadata_handler_class = cls._metadata_handler_class - if metadata_handler_class is None: - raise RuntimeError( - f"{cls.__name__} missing _metadata_handler_class for detection" - ) - - metadata_handler = metadata_handler_class(filemanager) - try: - metadata_handler.find_metadata_file(plate_folder) - except (MetadataNotFoundError, FileNotFoundError, TypeError): - return False - return True - @property @abstractmethod def root_dir(self) -> str: @@ -454,9 +476,12 @@ def _register_virtual_workspace_backend( """ from polystore.virtual_workspace import VirtualWorkspaceBackend from openhcs.constants.constants import Backend + from openhcs.core.virtual_workspace_metadata import METADATA_CONFIG # Always create a new backend for this plate (VirtualWorkspace is plate-specific) - backend = VirtualWorkspaceBackend(plate_root=Path(plate_path)) + backend = VirtualWorkspaceBackend( + plate_root=Path(plate_path), metadata_config=METADATA_CONFIG + ) filemanager.register_backend(Backend.VIRTUAL_WORKSPACE.value, backend) logger.info(f"Registered virtual workspace backend for {plate_path}") @@ -935,6 +960,7 @@ def create_microscope_handler( plate_folder, filemanager, allowed_types=allowed_auto_types, + source_bindings_config=source_bindings, ) if detected_microscope_type is None: if source_bindings is None or source_bindings.is_empty: @@ -1020,6 +1046,7 @@ def _auto_detect_microscope_type( plate_folder: Path, filemanager: FileManager, allowed_types: Optional[List[str]] = None, + source_bindings_config: Optional["SourceBindingsConfig"] = None, ) -> Optional[str]: """ Auto-detect microscope type using registry iteration. @@ -1060,7 +1087,9 @@ def _auto_detect_microscope_type( for handler_name in detection_order: handler_class = MICROSCOPE_HANDLERS[handler_name] - if handler_class.detect(plate_folder, filemanager): + if handler_class.detect( + plate_folder, filemanager, source_bindings_config + ): logger.info(f"Auto-detected {handler_name} microscope type") return handler_name logger.debug(f"{handler_name} metadata not found in {plate_folder}") diff --git a/openhcs/microscopes/source_bindings_handler.py b/openhcs/microscopes/source_bindings_handler.py index d6af813a8a..7178103625 100644 --- a/openhcs/microscopes/source_bindings_handler.py +++ b/openhcs/microscopes/source_bindings_handler.py @@ -115,8 +115,13 @@ def compatible_backends(self) -> list[Backend]: return [Backend.DISK] @classmethod - def detect(cls, plate_folder: Path, filemanager: FileManager) -> bool: - del plate_folder, filemanager + def detect( + cls, + plate_folder: Path, + filemanager: FileManager, + source_bindings_config: SourceBindingsConfig | None = None, + ) -> bool: + del plate_folder, filemanager, source_bindings_config return False def _replay_persisted_workspace( diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index cfa1841f98..faff29684a 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -3030,7 +3030,7 @@ def _write_roi_zip( ) summary = ( - f"Segmentation ROIs: {total_roi_count} cells\n" + f"Segmentation ROIs: {total_roi_count} parent labels\n" "Spatial dimensions: 2D\n" f"Projected source planes: {len(materialization_input.items)}\n" ) diff --git a/tests/unit/agent/test_synthetic_plate_zero_geometry_journey.py b/tests/unit/agent/test_synthetic_plate_zero_geometry_journey.py new file mode 100644 index 0000000000..439d49911b --- /dev/null +++ b/tests/unit/agent/test_synthetic_plate_zero_geometry_journey.py @@ -0,0 +1,94 @@ +"""Continuous real MCP generate/inspect/sample for zero-valued geometry.""" + +import json +from pathlib import Path + +import pytest + +from openhcs.mcp.dev_client import McpDevClient + + +def _payload(client, argv): + result = client.execute([*argv, "--json"], timeout_seconds=30) + assert result.returncode == 0, result.rendered_output + payload = result.payload["results"][0]["payloads"][0] + assert not payload.get("errors"), payload + return payload + + +@pytest.mark.parametrize("format", ("ImageXpress", "OperaPhenix")) +@pytest.mark.parametrize("native", (False, True)) +def test_zero_geometry_real_mcp_journey(tmp_path, monkeypatch, format, native): + monkeypatch.setenv("OPENHCS_CPU_ONLY", "true") + monkeypatch.setenv("OPENHCS_AGENT_READ_ROOTS", str(tmp_path)) + monkeypatch.setenv("OPENHCS_AGENT_WRITE_ROOTS", str(tmp_path)) + # Explicit owned stdio process: do not reuse or mutate a resident server. + with McpDevClient(use_resident_server=False) as client: + _payload(client, ["health"]) + _payload(client, ["authoring-context", "first_use"]) + root = tmp_path / "plate" + arguments = [ + "generate-synthetic-plate", + str(root), + "--grid-rows", + "1", + "--grid-cols", + "1", + "--tile-width", + "64", + "--tile-height", + "64", + "--overlap-percent", + "0", + "--stage-error-px", + "0", + "--wavelengths", + "1", + "--z-stack-levels", + "1", + "--num-cells", + "4", + "--well", + "A01", + "--random-seed", + "7", + "--format", + format, + ] + if native: + arguments.append("--openhcs-format") + generated = _payload(client, arguments) + assert generated["image_count"] == 1 + assert generated["overlap_percent"] == generated["stage_error_px"] == 0 + inspected = _payload(client, ["inspect-plate", str(root)]) + assert inspected["image_files"]["count"] == 1 + sampled = _payload( + client, + [ + "sample-plate-image", + str(root), + generated["sampled_image_files"][0], + "--height", + "8", + "--width", + "8", + "--include-array-values", + ], + ) + assert sampled["sample_shape"] == [8, 8] + assert sampled["dtype"] == "uint16" + assert sampled["sample_included"] + assert sampled["sample_values"] + if native: + metadata = json.loads(Path(generated["metadata_file_path"]).read_text()) + assert ( + sum( + len(subdir["image_files"]) + for subdir in metadata["subdirectories"].values() + ) + == 1 + ) + assert all( + subdir["grid_dimensions"] == [1, 1] + for subdir in metadata["subdirectories"].values() + ) diff --git a/tests/unit/metadata_namespace_327_worker.py b/tests/unit/metadata_namespace_327_worker.py new file mode 100644 index 0000000000..4f2fe46759 --- /dev/null +++ b/tests/unit/metadata_namespace_327_worker.py @@ -0,0 +1,45 @@ +"""Native process recipient; import dependency before application on purpose.""" + +import pickle +import sys +from pathlib import Path + +import numpy as np +import polystore.metadata_writer + +import openhcs + + +def main(): + from polystore.base import ImageSamplingRequest + + assert ( + Path(openhcs.__file__).resolve().parents[1] + == Path(__file__).resolve().parents[2] + ) + assert ( + Path(polystore.metadata_writer.__file__).resolve() + == Path(sys.argv[2]).resolve() + ) + with Path(sys.argv[1]).open("rb") as stream: + manager, root, config, expected = pickle.load(stream) + workspace = manager.registry["virtual_workspace"] + assert workspace.metadata_config == config + assert workspace.metadata_config.__class__ is config.__class__ + assert workspace._registry is manager.registry + np.testing.assert_array_equal( + manager.load(root / "virtual.npy", backend="virtual_workspace"), expected + ) + sample = manager.sample( + root / "virtual.npy", + backend="virtual_workspace", + request=ImageSamplingRequest(origin_yx=(1, 1), shape_yx=(2, 3)), + ) + np.testing.assert_array_equal(sample.data, expected[1:3, 1:4]) + assert sample.source_shape == expected.shape + assert workspace._resolve_ref(root / "virtual.npy").source_axis_indices == (1,) + print("native handoff: exact application namespace, pixels and provenance") + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_bioformats_preparation_selection.py b/tests/unit/test_bioformats_preparation_selection.py new file mode 100644 index 0000000000..2153a60d86 --- /dev/null +++ b/tests/unit/test_bioformats_preparation_selection.py @@ -0,0 +1,274 @@ +"""Known physical-source selections reach discovery before container opens.""" + +import json +from pathlib import Path + +import numpy as np +import pytest +import tifffile +from objectstate.lazy_factory import ensure_global_config_context +from polystore.bioformats_java import BioFormatsJavaContext + +from openhcs.constants.constants import AllComponents, Microscope +from openhcs.core.config import ( + GlobalPipelineConfig, + LazySourceBindingsConfig, + PipelineConfig, +) +from openhcs.core.image_file_serialization import TiffImageFileFormat +from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator +from openhcs.core.source_bindings import ( + ImagePlaneSource, + MetadataSelector, + NamedSourceBinding, + SourceBindingsConfig, + SourceFilterClause, + SourceFilterMatchType, + SourceFilterSubject, + SourceSelector, +) +from openhcs.microscopes.bioformats import BioFormatsHandler +from openhcs.microscopes.bioformats_adapter import BioFormatsJavaAdapter +from tests.unit.bioformats_fixture import bioformats_filemanager +from tests.unit.test_bioformats_java_adapter import ( + FakeBioFormatsContext, + FakeBioFormatsMetadata, +) + + +def _file_filter(name): + return SourceFilterClause( + SourceFilterSubject.FILE, SourceFilterMatchType.EQUALS, name + ) + + +class _SelectionContext(FakeBioFormatsContext): + """Controlled decoder response, preserving the real discovery/workspace path.""" + + single_file = True + + def is_single_file(self, path): + return self.single_file + + +@pytest.mark.parametrize("selection", ("global", "binding")) +@pytest.mark.parametrize("entrypoint", ("handler", "orchestrator", "auto_orchestrator")) +def test_selected_container_opens_once_and_projects_both_planes( + tmp_path, monkeypatch, selection, entrypoint +): + names = [f"container-{index:02}.czi" for index in range(16)] + metadata = {} + for index, name in enumerate(names): + (tmp_path / name).touch() + metadata[name] = FakeBioFormatsMetadata( + well_column=index, + well_id=f"Well:0:{index}", + sample_id=f"WellSample:{index}", + image_id=f"Image:{index}", + ) + context = _SelectionContext(metadata, declared_suffixes=(".czi",)) + opened = [] + original_open = context.open_reader + + def counted_open(path): + opened.append(Path(path).name) + return original_open(path) + + monkeypatch.setattr(context, "open_reader", counted_open) + monkeypatch.setattr( + BioFormatsJavaContext, "instance", classmethod(lambda cls: context) + ) + filters = (_file_filter(names[7]),) + bindings = SourceBindingsConfig( + source_filters=filters if selection == "global" else (), + bindings=( + NamedSourceBinding( + alias="selected", + selector=SourceSelector( + filters=filters if selection == "binding" else (), + ), + ), + ), + ) + filemanager = bioformats_filemanager() + if entrypoint == "handler": + handler = BioFormatsHandler(filemanager, source_bindings_config=bindings) + assert handler.initialize_workspace(tmp_path, filemanager) == tmp_path + assert opened == [names[7]] + else: + ensure_global_config_context(GlobalPipelineConfig, GlobalPipelineConfig()) + orchestrator = PipelineOrchestrator( + tmp_path, + storage_registry=filemanager.registry, + pipeline_config=PipelineConfig( + microscope=Microscope.AUTO + if entrypoint == "auto_orchestrator" + else Microscope.BIOFORMATS, + source_bindings_config=LazySourceBindingsConfig( + source_filters=bindings.source_filters, + bindings=bindings.bindings, + ), + ), + ).initialize() + assert orchestrator.is_initialized() + assert opened == [names[7]] * (2 if entrypoint == "auto_orchestrator" else 1) + assert len(orchestrator.get_component_keys(AllComponents.WELL)) == 1 + workspace = json.loads((tmp_path / "openhcs_metadata.json").read_text()) + mapping = workspace["subdirectories"]["."]["workspace_mapping"] + assert len(mapping) == 2 + assert all(names[7] in ref["backend_address"] for ref in mapping.values()) + + +def test_real_tiff_orchestrator_only_decodes_selected_source(tmp_path, monkeypatch): + names = [f"source-{index:02}.tif" for index in range(16)] + for index, name in enumerate(names): + tifffile.imwrite(tmp_path / name, np.full((8, 8), index, dtype=np.uint16)) + decoded = [] + original_read = TiffImageFileFormat.read + + def counted_read(self, path): + decoded.append(Path(path).name) + return original_read(self, path) + + monkeypatch.setattr(TiffImageFileFormat, "read", counted_read) + ensure_global_config_context(GlobalPipelineConfig, GlobalPipelineConfig()) + filemanager = bioformats_filemanager() + orchestrator = PipelineOrchestrator( + tmp_path, + storage_registry=filemanager.registry, + pipeline_config=PipelineConfig( + microscope=Microscope.AUTO, + source_bindings_config=LazySourceBindingsConfig( + bindings=( + NamedSourceBinding( + alias="selected", + selector=SourceSelector(filters=(_file_filter(names[7]),)), + ), + ), + ), + ), + ).initialize() + assert orchestrator.is_initialized() + assert decoded == [names[7], names[7]] + workspace = json.loads((tmp_path / "openhcs_metadata.json").read_text()) + (reference,) = workspace["subdirectories"]["."]["workspace_mapping"].values() + assert reference["backend_address"] == names[7] + pixels = filemanager.load(str(tmp_path / reference["backend_address"]), "disk") + np.testing.assert_array_equal(pixels, np.full((8, 8), 7, dtype=np.uint16)) + + +def test_companion_path_selection_does_not_discard_its_container(tmp_path, monkeypatch): + (tmp_path / "plate.fake").touch() + context = _SelectionContext() + context.single_file = False + monkeypatch.setattr( + BioFormatsJavaContext, "instance", classmethod(lambda cls: context) + ) + config = SourceBindingsConfig(source_filters=(_file_filter("well-a01.tif"),)) + (dataset,) = BioFormatsJavaAdapter(config).discover_stores(tmp_path) + assert len(dataset.candidates) == 2 + assert all( + "well-a01.tif" in c.source_filter_path_identities() for c in dataset.candidates + ) + + +def test_global_companion_filter_is_applied_after_container_decode(tmp_path, monkeypatch): + names = ("selected.fake", "unrelated.fake") + metadata = {} + for index, name in enumerate(names): + (tmp_path / name).touch() + metadata[name] = FakeBioFormatsMetadata( + well_column=index, + well_id=f"Well:0:{index}", + sample_id=f"WellSample:{index}", + image_id=f"Image:{index}", + ) + context = _SelectionContext(metadata) + context.single_file = False + original_open = context.open_reader + + def with_companion(path): + opened = original_open(path) + opened.reader.used_files = (Path(path).name, f"{Path(path).stem}.tif") + return opened + + monkeypatch.setattr(context, "open_reader", with_companion) + monkeypatch.setattr( + BioFormatsJavaContext, "instance", classmethod(lambda cls: context) + ) + filemanager = bioformats_filemanager() + handler = BioFormatsHandler( + filemanager, + source_bindings_config=SourceBindingsConfig( + source_filters=(_file_filter("selected.tif"),) + ), + ) + handler.initialize_workspace(tmp_path, filemanager) + workspace = json.loads((tmp_path / "openhcs_metadata.json").read_text()) + mapping = workspace["subdirectories"]["."]["workspace_mapping"] + assert len(mapping) == 2 + assert all("selected.fake" in ref["backend_address"] for ref in mapping.values()) + + +def test_binding_selection_is_union_and_metadata_is_not_guessed(tmp_path): + bindings = SourceBindingsConfig( + bindings=( + NamedSourceBinding( + alias="first", selector=SourceSelector(filters=(_file_filter("a.czi"),)) + ), + NamedSourceBinding( + alias="second", + selector=SourceSelector(filters=(_file_filter("b.czi"),)), + ), + ) + ) + assert bindings.discovery_path_matches(tmp_path, tmp_path / "a.czi") + assert bindings.discovery_path_matches(tmp_path, tmp_path / "b.czi") + assert not bindings.discovery_path_matches(tmp_path, tmp_path / "c.czi") + unrestricted = SourceBindingsConfig( + bindings=(*bindings.bindings, NamedSourceBinding(alias="all")) + ) + assert unrestricted.discovery_path_matches(tmp_path, tmp_path / "c.czi") + + +def test_explicit_source_and_metadata_only_discovery_stay_declared(tmp_path): + selected = tmp_path / "selected.czi" + selected.touch() + exact = SourceBindingsConfig( + bindings=( + NamedSourceBinding( + alias="exact", explicit_source=ImagePlaneSource(uri="selected.czi") + ), + ) + ) + assert exact.discovery_path_matches(tmp_path, selected) + assert not exact.discovery_path_matches(tmp_path, tmp_path / "unrelated.czi") + metadata_only = SourceBindingsConfig( + bindings=( + NamedSourceBinding( + alias="decoded", + selector=SourceSelector(metadata=(MetadataSelector("well", "A01"),)), + ), + ) + ) + # The filename cannot establish which decoded well this container supplies. + assert metadata_only.discovery_path_matches(tmp_path, tmp_path / "unrelated.czi") + + +def test_discovery_reuses_directory_or_group_filter_semantics(tmp_path): + filters = ( + SourceFilterClause( + SourceFilterSubject.DIRECTORY, SourceFilterMatchType.CONTAINS, "selected" + ), + SourceFilterClause( + SourceFilterSubject.FILE, SourceFilterMatchType.EQUALS, "a.czi", any_group=0 + ), + SourceFilterClause( + SourceFilterSubject.FILE, SourceFilterMatchType.EQUALS, "b.czi", any_group=0 + ), + ) + config = SourceBindingsConfig(source_filters=filters) + assert config.discovery_path_matches(tmp_path, tmp_path / "selected" / "a.czi") + assert config.discovery_path_matches(tmp_path, tmp_path / "selected" / "b.czi") + assert not config.discovery_path_matches(tmp_path, tmp_path / "selected" / "c.czi") + assert not config.discovery_path_matches(tmp_path, tmp_path / "other" / "a.czi") diff --git a/tests/unit/test_fragmented_roi_materialization.py b/tests/unit/test_fragmented_roi_materialization.py new file mode 100644 index 0000000000..a65486534d --- /dev/null +++ b/tests/unit/test_fragmented_roi_materialization.py @@ -0,0 +1,119 @@ +"""Fragmented diagnostic labels retain parent identity across real materialization.""" + +import json +from collections import Counter +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pytest +import tifffile +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import ROI_ZIP_METADATA_MEMBER, load_rois_from_zip + +from openhcs.core.runtime_object_labels import ( + ObjectLabelPayload, + ObjectLabelVariantData, +) +from openhcs.core.source_spatial_domain import SourceSpatialDomain +from openhcs.processing.materialization import ( + ImageFileOptions, + MaterializationSpec, + ROIOptions, + materialization_outputs, +) + + +def fragmented_labels(): + labels = np.zeros((128, 128), dtype=np.int32) + labels[4:124:4, 4:124:4] = 188 + labels[1:3, 1:3] = 7 + labels[125:127, 125:127] = 42 + return labels + + +def _payload(labels=None): + return ObjectLabelPayload( + variant_data=ObjectLabelVariantData( + labels=fragmented_labels() if labels is None else labels + ), + source_path="/synthetic/raw.tif", + source_component_metadata={"well": "A01", "channel": 1}, + source_spatial_domain=SourceSpatialDomain( + origin_yx=(10, 20), + source_shape_yx=(256, 256), + ), + ) + + +def test_archive_fragments_keep_parent_label_and_source_geometry(tmp_path): + filemanager = FileManager({"disk": DiskStorageBackend()}) + outputs = materialization_outputs( + MaterializationSpec(ROIOptions(min_area=1)), + _payload(), + str(tmp_path / "fragmented"), + filemanager, + ) + archive = next(output for output in outputs if output.path.endswith(".roi.zip")) + summary = next(output for output in outputs if output.path.endswith(".txt")) + assert "Segmentation ROIs: 3 parent labels\n" in summary.content + parents = archive.content + assert Counter(roi.metadata["label"] for roi in parents) == {7: 1, 42: 1, 188: 1} + assert Counter({roi.metadata["label"]: len(roi.shapes) for roi in parents}) == { + 7: 1, + 42: 1, + 188: 900, + } + filemanager.save(parents, archive.path, "disk") + with ZipFile(archive.path) as zipped: + members = [name for name in zipped.namelist() if name.endswith(".roi")] + assert len(members) == 902 + metadata = json.loads(zipped.read(ROI_ZIP_METADATA_MEMBER)) + assert Counter(value["label"] for value in metadata.values()) == { + 7: 1, + 42: 1, + 188: 900, + } + # Native reopen flattens shapes, but every member still owns its parent label. + restored = load_rois_from_zip(Path(archive.path)) + assert Counter(roi.metadata["label"] for roi in restored) == {7: 1, 42: 1, 188: 900} + fragmented = next(roi for roi in restored if roi.metadata["label"] == 188) + assert fragmented.metadata["area"] == 900 + assert fragmented.metadata["centroid"] == (72.0, 82.0) + coordinates = np.concatenate( + [roi.shapes[0].coordinates for roi in restored if roi.metadata["label"] == 188] + ) + assert coordinates[:, 0].min() == pytest.approx(13.5) + assert coordinates[:, 0].max() == pytest.approx(130.5) + assert coordinates[:, 1].min() == pytest.approx(23.5) + assert coordinates[:, 1].max() == pytest.approx(140.5) + + +@pytest.mark.parametrize("with_holes_and_borders", (False, True)) +def test_declared_raster_selection_preserves_parent_labels_without_roi_extraction( + tmp_path, monkeypatch, with_holes_and_borders +): + monkeypatch.setattr( + "polystore.roi.extract_rois_from_labeled_mask", + lambda *args, **kwargs: pytest.fail("Unselected ROI extraction ran"), + ) + labels = fragmented_labels() + if with_holes_and_borders: + labels[:32, :32] = 7 + labels[8:24, 8:24] = 0 + labels[12:16, 12:16] = 42 + labels[-1, -1] = 188 + filemanager = FileManager({"disk": DiskStorageBackend()}) + outputs = materialization_outputs( + MaterializationSpec( + ROIOptions(min_area=1), ImageFileOptions(filename_suffix=".tif") + ), + _payload(labels), + str(tmp_path / "fragmented"), + filemanager, + output_path_filter=lambda path: path.suffix == ".tif", + ) + assert len(outputs) == 1 + filemanager.save(outputs[0].content, outputs[0].path, "disk") + np.testing.assert_array_equal(tifffile.imread(outputs[0].path), labels) diff --git a/tests/unit/test_metadata_namespace_327.py b/tests/unit/test_metadata_namespace_327.py new file mode 100644 index 0000000000..5df136b5fd --- /dev/null +++ b/tests/unit/test_metadata_namespace_327.py @@ -0,0 +1,103 @@ +"""The application metadata declaration participates in the generic contract.""" + +import ast +import json +import pickle +import subprocess +import sys +from pathlib import Path + +import numpy as np +import polystore.metadata_writer +import pytest +from polystore.disk import DiskBackend +from polystore.filemanager import FileManager +from polystore.metadata_writer import METADATA_CONFIG as DEPENDENCY_CONFIG +from polystore.metadata_writer import MetadataConfig +from polystore.virtual_workspace import SourcePixelRef, VirtualWorkspaceBackend + +from openhcs.core.virtual_workspace_metadata import ( + METADATA_CONFIG, + AtomicMetadataWriter, + OpenHCSMetadataConfig, +) + + +@pytest.mark.parametrize( + "config", + [ + METADATA_CONFIG, + OpenHCSMetadataConfig(METADATA_FILENAME="application-custom.json"), + ], + ids=["declared-application", "explicit-custom"], +) +def test_application_namespace_persist_reopen_and_native_handoff( + tmp_path: Path, config +): + source = np.arange(2 * 4 * 5, dtype=np.uint16).reshape(2, 4, 5) + np.save(tmp_path / "source.npy", source) + ref = SourcePixelRef("disk", "source.npy", (1,)) + AtomicMetadataWriter().merge_subdirectory_metadata( + config.metadata_path(tmp_path), + { + ".": {"workspace_mapping": {"virtual.npy": ref.to_workspace_mapping()}}, + }, + ) + manager = FileManager( + { + "disk": DiskBackend(), + "virtual_workspace": VirtualWorkspaceBackend( + tmp_path, metadata_config=config + ), + } + ) + np.testing.assert_array_equal( + manager.load(tmp_path / "virtual.npy", backend="virtual_workspace"), source[1] + ) + handoff = tmp_path / "handoff.pickle" + with handoff.open("wb") as stream: + pickle.dump((manager, tmp_path, config, source[1]), stream) + result = subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name("metadata_namespace_327_worker.py")), + str(handoff), + polystore.metadata_writer.__file__, + ], + text=True, + capture_output=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "exact application namespace" in result.stdout + assert not (tmp_path / DEPENDENCY_CONFIG.METADATA_FILENAME).exists() + assert ( + json.loads(config.metadata_path(tmp_path).read_text())["subdirectories"]["."][ + "workspace_mapping" + ]["virtual.npy"] + == ref.to_workspace_mapping() + ) + + +def test_application_declaration_inherits_the_existing_metadata_contract(): + assert isinstance(METADATA_CONFIG, MetadataConfig) + assert OpenHCSMetadataConfig.metadata_path is MetadataConfig.metadata_path + assert OpenHCSMetadataConfig.managed_paths is MetadataConfig.managed_paths + + +def test_application_custom_environment_is_independent_of_dependency(monkeypatch): + monkeypatch.setenv("OPENHCS_METADATA_FILENAME", "explicit-openhcs.json") + monkeypatch.setenv("POLYSTORE_METADATA_FILENAME", "explicit-polystore.json") + assert OpenHCSMetadataConfig().METADATA_FILENAME == "explicit-openhcs.json" + assert MetadataConfig().METADATA_FILENAME == "explicit-polystore.json" + + +def test_application_bootstrap_does_not_set_the_dependency_namespace(): + import openhcs + + tree = ast.parse(Path(openhcs.__file__).read_text()) + assert not any( + isinstance(node, ast.Constant) and node.value == "POLYSTORE_METADATA_FILENAME" + for node in ast.walk(tree) + ) diff --git a/tests/unit/test_microscope_handler_identity.py b/tests/unit/test_microscope_handler_identity.py index 5aa119b3d8..8c3203472d 100644 --- a/tests/unit/test_microscope_handler_identity.py +++ b/tests/unit/test_microscope_handler_identity.py @@ -20,9 +20,12 @@ from openhcs.core.steps.function_step import FunctionStep from openhcs.microscopes import create_microscope_handler, get_all_handler_types from openhcs.microscopes.microscope_base import ( + MetadataMicroscopeDetector, MicroscopeHandler, MicroscopeSourceSelectionRole, ) +from openhcs.microscopes.imagexpress import ImageXpressHandler +from openhcs.microscopes.openhcs import OpenHCSMicroscopeHandler from openhcs.microscopes.opera_phenix import OperaPhenixHandler from openhcs.processing.backends.processors.numpy_processor import percentile_normalize from openhcs.demo.synthetic_data import ( @@ -69,6 +72,52 @@ def test_typed_microscope_values_are_exact_registered_handler_keys() -> None: } == configured_types +@pytest.mark.parametrize( + "handler_type", + (ImageXpressHandler, OperaPhenixHandler, OpenHCSMicroscopeHandler), +) +@pytest.mark.parametrize("missing", (False, True)) +def test_metadata_detection_uses_each_declared_owner( + monkeypatch, tmp_path: Path, handler_type, missing: bool, +) -> None: + from polystore.exceptions import MetadataNotFoundError + + observed = [] + filemanager = bioformats_filemanager() + metadata_type = handler_type._metadata_handler_class + + def find_metadata_file(metadata, plate_folder): + observed.append((type(metadata), plate_folder)) + if missing: + raise MetadataNotFoundError("declared metadata unavailable") + return plate_folder / "declared-metadata" + + monkeypatch.setattr(metadata_type, "find_metadata_file", find_metadata_file) + + assert handler_type.detect(tmp_path, filemanager) is not missing + assert observed == [(metadata_type, tmp_path)] + + +def test_metadata_detection_new_case_needs_only_its_declaration( + monkeypatch, tmp_path: Path, +) -> None: + metadata_type = ImageXpressHandler._metadata_handler_class + + class NewMetadataDetector(MetadataMicroscopeDetector): + _metadata_handler_class = metadata_type + + observed = [] + + def find_metadata_file(metadata, plate_folder): + observed.append((type(metadata), plate_folder)) + return plate_folder / "new-case-metadata" + + monkeypatch.setattr(metadata_type, "find_metadata_file", find_metadata_file) + + assert NewMetadataDetector.detect(tmp_path, bioformats_filemanager()) is True + assert observed == [(metadata_type, tmp_path)] + + def test_config_schema_patch_and_source_share_opera_handler_identity() -> None: service = ConfigService() schema = service.describe_schema("pipeline") diff --git a/tests/unit/test_source_binding_path_admission.py b/tests/unit/test_source_binding_path_admission.py new file mode 100644 index 0000000000..7a4359a886 --- /dev/null +++ b/tests/unit/test_source_binding_path_admission.py @@ -0,0 +1,244 @@ +"""Shared declaration-owned path admission before and after source decoding.""" + +import ast +from pathlib import Path + +import pytest +from polystore.virtual_workspace import SourcePixelRef + +from openhcs.constants.constants import AllComponents +from openhcs.core.source_binding_workspace import SourceBindingWorkspaceProjector +from openhcs.core.source_bindings import ( + ComponentSelector, + ImagePlaneSource, + MetadataSelector, + NamedSourceBinding, + SourceBindingsConfig, + SourceFilterClause, + SourceFilterMatchType, + SourceFilterSubject, + SourceSelector, +) +from openhcs.core.source_projection import SourceCandidate + + +def _candidate(path, *, identities=(), metadata=None): + return SourceCandidate( + source_ref=SourcePixelRef("disk", str(path)), + relative_path=str(path), + source_filter_paths=identities, + metadata=metadata or {}, + ) + + +@pytest.mark.parametrize("absolute_candidate", (False, True)) +@pytest.mark.parametrize("explicit_uri", ("relative", "absolute", "file_uri")) +def test_exact_source_admission_agrees_before_and_after_decode( + tmp_path, absolute_candidate, explicit_uri +): + selected = tmp_path / "selected.czi" + selected.touch() + uris = { + "relative": selected.name, + "absolute": str(selected), + "file_uri": selected.as_uri(), + } + binding = NamedSourceBinding( + alias="selected", explicit_source=ImagePlaneSource(uri=uris[explicit_uri]) + ) + config = SourceBindingsConfig(bindings=(binding,)) + projector = SourceBindingWorkspaceProjector(config) + for path, expected in ((selected, True), (tmp_path / "other.czi", False)): + candidate = _candidate(path if absolute_candidate else path.name) + assert config.discovery_path_matches(tmp_path, path) is expected + assert ( + projector.candidate_matches_binding(candidate, binding, tmp_path) + is expected + ) + + +@pytest.mark.parametrize( + "relative_path, expected", + ( + ("selected/a.czi", True), + ("selected/b.czi", True), + ("selected/c.czi", False), + ("other/a.czi", False), + ), +) +def test_binding_directory_regex_and_or_group_use_the_same_owner( + tmp_path, relative_path, expected +): + binding = NamedSourceBinding( + alias="selected", + selector=SourceSelector( + filters=( + SourceFilterClause( + SourceFilterSubject.DIRECTORY, + SourceFilterMatchType.CONTAINS_REGEX, + r"(?:^|/)selected$", + ), + SourceFilterClause( + SourceFilterSubject.FILE, + SourceFilterMatchType.EQUALS, + "a.czi", + any_group=0, + ), + SourceFilterClause( + SourceFilterSubject.FILE, + SourceFilterMatchType.EQUALS, + "b.czi", + any_group=0, + ), + ) + ), + ) + config = SourceBindingsConfig(bindings=(binding,)) + candidate = _candidate(relative_path) + assert config.discovery_path_matches(tmp_path, tmp_path / relative_path) is expected + assert ( + SourceBindingWorkspaceProjector(config).candidate_matches_binding( + candidate, binding, tmp_path + ) + is expected + ) + + +@pytest.mark.parametrize( + "metadata, expected", + ( + ({"well": "A01", "channel": "1"}, True), + ({"well": "B01", "channel": "1"}, False), + ({"well": "A01", "channel": "2"}, False), + ({}, False), + ), +) +def test_physical_admission_does_not_predict_decoded_metadata_or_components( + tmp_path, metadata, expected +): + binding = NamedSourceBinding( + alias="selected", + selector=SourceSelector( + metadata=(MetadataSelector("well", "A01"),), + components=(ComponentSelector(AllComponents.CHANNEL, "1"),), + ), + ) + config = SourceBindingsConfig(bindings=(binding,)) + assert config.discovery_path_matches(tmp_path, tmp_path / "unknown.czi") + assert ( + SourceBindingWorkspaceProjector(config).candidate_matches_binding( + _candidate("unknown.czi", metadata=metadata), binding, tmp_path + ) + is expected + ) + + +def test_decoded_companion_filters_do_not_replace_exact_entrypoint_identity(tmp_path): + entrypoint = tmp_path / "plate.fake" + companion = tmp_path / "well-a01.tif" + entrypoint.touch() + companion.touch() + selector = SourceSelector( + filters=( + SourceFilterClause( + SourceFilterSubject.FILE, + SourceFilterMatchType.EQUALS, + companion.name, + ), + ) + ) + candidate = _candidate( + entrypoint.name, identities=(entrypoint.name, companion.name) + ) + for exact_source, expected in ( + (None, True), + (entrypoint.name, True), + (companion.name, False), + ): + binding = NamedSourceBinding( + alias="selected", + selector=selector, + explicit_source=None + if exact_source is None + else ImagePlaneSource(uri=exact_source), + ) + config = SourceBindingsConfig(bindings=(binding,)) + # Discovery cannot see companions; the adapter must keep compound readers. + assert not config.discovery_path_matches(tmp_path, entrypoint) + assert ( + SourceBindingWorkspaceProjector(config).candidate_matches_binding( + candidate, binding, tmp_path + ) + is expected + ) + + +def test_unrestricted_alias_keeps_union_without_erasing_other_alias_filters(tmp_path): + selected = NamedSourceBinding( + alias="selected", + selector=SourceSelector( + filters=( + SourceFilterClause( + SourceFilterSubject.FILE, + SourceFilterMatchType.EQUALS, + "selected.czi", + ), + ) + ), + ) + unrestricted = NamedSourceBinding(alias="all") + config = SourceBindingsConfig(bindings=(selected, unrestricted)) + other = _candidate("other.czi") + assert config.discovery_path_matches(tmp_path, tmp_path / other.relative_path) + projector = SourceBindingWorkspaceProjector(config) + assert not projector.candidate_matches_binding(other, selected, tmp_path) + assert projector.candidate_matches_binding(other, unrestricted, tmp_path) + + +def test_preparation_consumers_cannot_reauthor_binding_or_java_lifetime(): + """Guard the two reviewed owner bypasses, without importing a Java runtime.""" + root = Path(__file__).resolve().parents[2] + for path, class_name, method_name, owner_call, forbidden in ( + ( + "openhcs/core/source_bindings.py", + "SourceBindingsConfig", + "discovery_path_matches", + "physical_path_matches", + {"explicit_source", "selector"}, + ), + ( + "openhcs/core/source_binding_workspace.py", + "SourceBindingWorkspaceProjector", + "candidate_matches_binding", + "physical_path_matches", + {"explicit_source", "filters"}, + ), + ( + "openhcs/microscopes/bioformats_adapter.py", + "BioFormatsJavaAdapter", + "discover_stores", + "is_single_file", + {"ImageReader", "ensure_initialized", "isSingleFile"}, + ), + ): + tree = ast.parse((root / path).read_text()) + owner = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ) + method = next( + node + for node in owner.body + if isinstance(node, ast.FunctionDef) and node.name == method_name + ) + attributes = { + node.attr for node in ast.walk(method) if isinstance(node, ast.Attribute) + } + assert owner_call in attributes, (path, method_name) + assert not attributes & forbidden, (path, method_name, attributes & forbidden) + if class_name == "BioFormatsJavaAdapter": + assert not any( + isinstance(node, ast.FunctionDef) and node.name == "_is_single_file" + for node in owner.body + ) diff --git a/tests/unit/test_synthetic_generator_zero_geometry.py b/tests/unit/test_synthetic_generator_zero_geometry.py new file mode 100644 index 0000000000..00a60746cb --- /dev/null +++ b/tests/unit/test_synthetic_generator_zero_geometry.py @@ -0,0 +1,64 @@ +"""Zero overlap/jitter retain literal geometry in both declared plate layouts.""" + +import numpy as np +import pytest +import tifffile + +from openhcs.demo.synthetic_data import SyntheticMicroscopyGenerator + + +@pytest.mark.parametrize("format", ("ImageXpress", "OperaPhenix")) +@pytest.mark.parametrize("native", (False, True)) +@pytest.mark.parametrize("grid", ((1, 1), (2, 2))) +def test_zero_geometry_generates_deterministic_requested_tiles( + tmp_path, format, native, grid +): + outputs = [] + for run in range(2): + root = tmp_path / str(run) + generator = SyntheticMicroscopyGenerator( + output_dir=str(root), + grid_size=grid, + tile_size=(64, 64), + overlap_percent=0, + stage_error_px=0, + wavelengths=1, + z_stack_levels=1, + num_cells=4, + wells=["A01"], + format=format, + openhcs_format=native, + random_seed=7, + ) + assert generator.image_size == (64 * grid[1], 64 * grid[0]) + for row in range(grid[0]): + for col in range(grid[1]): + assert generator._site_position(row, col) == (64 * col, 64 * row) + generator.generate_dataset() + images = { + path.relative_to(root): tifffile.imread(path) + for path in root.rglob("*.tif*") + } + assert len(images) == grid[0] * grid[1] + assert all(image.shape == (64, 64) for image in images.values()) + outputs.append(images) + assert outputs[0].keys() == outputs[1].keys() + for path in outputs[0]: + np.testing.assert_array_equal(outputs[0][path], outputs[1][path]) + + +def test_subpixel_overlap_does_not_sample_an_empty_pixel_region(tmp_path): + generator = SyntheticMicroscopyGenerator( + output_dir=str(tmp_path), + grid_size=(1, 1), + tile_size=(32, 64), + overlap_percent=1, + stage_error_px=0, + wavelengths=1, + z_stack_levels=1, + num_cells=4, + wells=["A01"], + random_seed=7, + ) + generator.generate_dataset() + assert len(list(tmp_path.rglob("*.tif"))) == 1