Select original declaration owners for knowledge conversion (#376)

docs/validation/diagnose-knowledge-376.py

@@ -0,0 +1,60 @@
+"""Source-only diagnostic: deny catalogue preparation and native execution."""
+
+import importlib.util
+import cProfile
+import pstats
+import sys
+import time
+import traceback
+from pathlib import Path
+
+# Reuse only the parent's existing ABI substrate, without source/install writes.
+import openhcs
+
+# The executable may resolve to a shared Python; the supplied readonly root is
+# explicit so it never guesses an installed incarnation from that executable.
+installed_package = Path(sys.argv[1])
+for module_name, relative in (
+    ("openhcs.core._tabular_native", "core/_tabular_native.abi3.so"),
+    (
+        "openhcs.processing.backends.cellprofiler._granularity_native",
+        "processing/backends/cellprofiler/_granularity_native.abi3.so",
+    ),
+):
+    spec = importlib.util.spec_from_file_location(module_name, installed_package / relative)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+
+started = time.monotonic()
+from openhcs.processing.backends.lib_registry.registry_service import RegistryService
+
+
+def denied_preparation(*args, **kwargs):
+    traceback.print_stack()
+    raise AssertionError("source retrieval attempted unrelated catalog preparation")
+
+
+RegistryService.prepare_persistent_catalog = denied_preparation
+RegistryService.prepare_in_current_process = denied_preparation
+
+print("registry imported", time.monotonic() - started, flush=True)
+from openhcs.agent.dto.knowledge import KnowledgeBaseDocumentRequest
+from openhcs.agent.services.knowledge_base_service import KnowledgeBaseService
+
+print("knowledge imported", time.monotonic() - started, flush=True)
+profile = cProfile.Profile()
+profile.enable()
+document = KnowledgeBaseService().get_document(
+    KnowledgeBaseDocumentRequest.from_fields(
+        document_id="openhcs_official30_benchmark_recipes",
+        section_id="cp-tutorial-pixel-based-classification-openhcs-python",
+        max_chars=50000,
+    )
+)
+profile.disable()
+print("completed", time.monotonic() - started, flush=True)
+print("errors", document.errors, flush=True)
+print("content_chars", len(document.content), flush=True)
+pstats.Stats(profile).sort_stats("cumulative").print_stats(35)
+assert not document.errors

docs/validation/knowledge-lazy-conversion-376.rst

@@ -0,0 +1,217 @@
+Selected knowledge conversion deadline: issue376
+================================================
+
+Status: bounded source diagnosis published in draftPR377; no repair or installed
+acceptance claim. Related canonical lookup defect is independently issue379.
+Owner: managed-viewer sidecar, now distinct knowledge lazy-conversion owner.
+Base: main6ace57656 in /home/ts/wt/openhcs-knowledge-lazy-conversion-20261001.
+
+Original installed failure
+--------------------------
+
+The read-only openhcs_get_knowledge_document request selects document
+openhcs_official30_benchmark_recipes, section
+cp-tutorial-pixel-based-classification-openhcs-python, max_chars50000.
+The original10s transport observation returns mcp_transport_failed/TimeoutError.
+Parent reports the same MCP PID1658909 remains healthy. No retry, restart,
+larger deadline or changed max_chars is authorized.
+
+Original evidence remains parent-owned in
+/home/ts/wt/openhcs-issue-batch-20260929/calibration371-classification372-installed-20261001:
+launch.sh, mcp.stdin line20 and mcp.stdout lines1429..1452.
+Private wheel SHA256:
+95123985e863847e7366f195194845f1b7ef9d661d04b254f11c49ea8e3eee33;
+parentff52f0e includes371/372.
+These references identify retained evidence, not a fresh execution.
+
+Required relation and ownership
+-------------------------------
+
+Selected source retrieval must use the existing importer and canonical public
+PipelineDocument renderer without initiating unrelated catalogue/runtime work.
+Trace KnowledgeBaseService._official30_source_section_lines ->
+_official30_public_source -> import_cellprofiler_pipeline ->
+PipelineDocumentAuthority. Do not replace that mechanism with another converter,
+raw cppipe fallback, generated-source cache or larger deadline.
+
+Applicable patterns: IMPL-12/13 (reuse the original translation/serialization
+owners), MEMB-1/2 (derive discovery from declarations and the existing registry),
+IMPL-3 (no generic consumer concrete-type dispatch). Shared algorithms stay on
+their owning ancestor; leaves supply only independent nominal hooks.
+
+Checked all open PR titles and matching knowledge issues. No matching owner
+found. Schrodinger PR358 owns cold runtime startup/readiness, not this boundary:
+ZMQClientService, ZMQExecutionClient and paired ZMQRuntime are excluded.
+If diagnosis reaches that seam, coordinate before edits. Parent371 calibration
+and372 classification source and frozen blind/runtime boundaries are excluded.
+Snapshot observation/writer and Qt binding hooks remain unchanged.
+
+Validation and disposition
+---------------------------
+
+Source-only: one CPU,512MiB combined RSS,60s per shard, no providers/plugins,
+native/MCP/UI/Fiji/download/install or heavy lock. Parent occupies the native
+slot and owns subsequent original installed10s entrypoint acceptance.
+Resource helper reports RAM20.0GiB, home7.6GiB and swap10.7GiB: no parallel fleet
+or large tests; only serial bounded source checks and small durable receipts.
+Owned scratch: /home/ts/.cache/agent-scratch/knowledge-lazy-conversion-376-20261001,
+for isolated cache/pytest fixtures and archive extraction verification only.
+Cleanup completed after every bounded source worker exited and fresh archive
+extraction matched: removed only that80KiB scratch directory and36KiB loose
+unversioned raw evidence copies. The byte-exact archived logs remain retained;
+parent evidence, source/worktrees, installed files and shared caches are intact.
+No complete global NRA/R1 claim.
+
+Done when the original selected request has focused behavioral evidence,
+an independent declaration/hooks case needs no consumer edits, missing and
+unsupported conversion remain typed, original failures are retained, original
+pinned R0 covers all changed production files, and parent installed acceptance
+has passed. Current checkpoint does not satisfy those conditions.
+
+Source diagnosis:376
+-------------------
+
+Actual original typed request ran on unchanged main production source. The
+diagnostic denied both RegistryService preparation entrypoints; neither was
+called. The first check returned10174 content characters without errors,
+9.99s total wall including source/ABI imports,394160KiB peak. A separately
+profiled cold process completed7.36s total /418172KiB; get_document itself
+took4.464s and selected source conversion3.992s. These successful source runs
+do not reproduce the parent's installed transport TimeoutError.
+
+The profile locates eager work at import_cellprofiler_pipeline ->
+CellProfilerModule.source_bindings_for_modules -> require_module -> for_module
+-> LazyDiscoveryDict.values/_discover -> discover_registry_classes. Full-family
+discovery took3.283s, including imports of unrelated alignment declarations.
+This is a proven cold-cost boundary, not proof that it alone caused the
+installed10s timeout. The converter already limits numerical translation to
+the selected manifest case; the generic global catalogue-preparation hypothesis
+was not observed and is not asserted here.
+
+The source regression forbids full backend discovery and invokes the original
+KnowledgeBaseService with the actual requested section and cached cppipe source.
+It fails with official30_conversion_failed, exposing the eager declaration
+lookup. No native application or scientific callable was invoked. The check
+does not substitute a converter or invent an alternative source document.
+
+Required owning seam: CellProfilerModule owns nominal module lookup;
+LazyDiscoveryDict owns discovery. Any selected-discovery repair must extend
+that owner, derive candidates from declarations, preserve normalized aliases
+and duplicate/unknown rejection, and retain one importer and public renderer.
+Do not put a module-name/filename roster, class switch or copied discovery
+procedure into KnowledgeBaseService. IMPL-13 and MEMB-1/2 apply. No production
+seam was changed in this diagnostic checkpoint, and no independent new-case
+success or global NRA completion is claimed.
+
+Related precise boundary:379
+---------------------------
+
+Parent remote registration succeeded once on owned5993 with canonical ID
+openhcs:engineering_calibration_classification_fixture and stable import
+openhcs.processing.custom_functions.engineering_calibration_classification_fixture.
+Persisted engineering source SHA256:
+3157e246de9348b0522f8e4d2712b84df06db71185d8c09ce868b6f74c7349ce.
+describe_function returned the required pixel_size and output contracts.
+inspect_pipeline_source_artifact_plan then returned pipeline_source_invalid_document
+with Unknown canonical function ID, before compilation. Original stdin29/31/36
+and stdout4640..4670 retain that failure. Parent's later stable-import spelling
+is a distinct source version, not evidence that the original canonical route
+passed. Original70502 handle and installed process remain parent-owned.
+
+The endpoint catalogue's get/reference methods use the remote incarnation and
+revision. In-process inspection calls PipelineDocumentAuthority.from_source;
+source get_function directly reads local RegistryService._metadata_cache via
+get_all_functions_with_metadata. Persisted registration intentionally does not
+publish ephemeral source locally. The original stable-import loader instead
+loads the persisted source through CustomFunctionManager and publishes through
+CustomFunctionRuntimeRegistry. An already cached local catalogue also fails
+to include a subsequently published custom declaration.
+
+The independent source witness declares its own @numpy function, projects and
+publishes it through the existing custom runtime owner, and never executes its
+body. With an existing empty local catalogue, get_function(metadata.composite_key)
+still raises Unknown canonical function ID. Cleanup removes only that
+process-local test declaration. This proves a custom declaration/cached lookup
+gap without remote registration or replay; it does not prove a remote-to-local
+transport repair. The unknown-ID assertion is presently unreachable after the
+first failure and is not counted as passing acceptance.
+
+BOUND-2: canonical lookup must not bypass CustomFunctionRuntimeRegistry's
+source-owned projection. MEMB-1/2: do not mirror endpoint membership into a new
+local cache. The next repair must decide source-versus-endpoint authority at
+the original lookup boundary, preserve revisions/ambiguity and exact namespace,
+and use the existing registry-family hooks rather than a consumer library switch.
+This is related ownership analysis, not a proven common timing cause for376.
+
+Retained checks and next boundary
+--------------------------------
+
+Original source regression result:2 failed,2 pytest configuration warnings,
+2.91s wall /177516KiB peak, exit1. Both failures are retained unchanged and are
+intentional failing behavioral witnesses; neither is marked xfail or waived.
+All three source shards used one CPU, kernel512MiB/no swap and timeout60s,
+provider/plugin-free. Shared dependencies and ABI binaries were read-only;
+the diagnostic explicitly loaded the two ABI modules from the existing
+generated-inputs-installed-parent environment without copying or installing.
+Actual metaclass dependency import paths are visible in the original profile.
+
+Byte-exact source logs, selected original installed stdin/stdout excerpts,
+diagnostic recipe and red test source are archived in
+docs/validation/issue376-source-diagnosis-20261001.tar.gz, SHA256:
+f9cbf53216cf89ce0ddd0c6ea80b5d44c35b4aa5d40f968c127215df37409321.
+Fresh extraction compared identical to every loose raw evidence file. Excerpts
+preserve original CRLF and whitespace; they are not a complete/final live
+session archive. Parent's original full logs remain unchanged. Raw whitespace
+is evidence, not rewritten to manufacture diff success.
+
+Changed paths are this receipt, the diagnostic source, the two red regression
+tests in one file, and the evidence archive. No production, dependency gitlink,
+startup/connect,371/372, snapshot, native runtime or installed source changed.
+There is no production R0 delta to measure yet. A future product repair requires
+original pinned R0 over every actual changed production file, not a copied
+detector or positive-delta waiver. Parent owns original installed10s acceptance.
+Current next work is selected declaration discovery/lookup ownership; the
+timeout's exact installed timing remains unqualified, not silently fixed.
+
+Additional installed witness
+-----------------------------
+
+Parent engineering evidence now records a second exact generated-source section
+timing out: knowledge-document openhcs_official30_benchmark_recipes,
+section examplehuman-openhcs-python, max_chars50000. It remains issue376 under
+this same PR377 owner, not a second feature or a case-name-specific workaround.
+
+Retained root:
+/home/ts/wt/openhcs-issue-batch-20260929/neurite-development-skill383-20261001/output.
+mcp.stdin line29 is the exact request; mcp.stdout2704..2726 contains the
+mcp_transport_failed/TimeoutError reply. author-record.rst's Example retrieval
+section explicitly keeps the timed-out read pending disposition, not replayed
+and not proof that conversion did no work. No retrieval parity is claimed.
+
+SHA256 of the byte-exact stdin line29, including its original line ending:
+20cdfc6bb9aec372d469ce527ab2aca32428ffdf5f1f62f8bf0df8e0f2d20cf4.
+SHA256 of the byte-exact stdout2704..2726 slice, including original CRLF:
+f1028363754d76ff325376032d79d22544276179b599c2131751d6912676018b.
+These hashes identify the original slices, not a complete/final session archive.
+No redundant snapshot, copied corpus or new archive was created for this update.
+
+The distinct subsequent corpus-map request at stdin30 succeeded:
+document openhcs_example_corpus_map, section
+openhcs-processing-presets-pipelines-loose-operaphenix-neurite-outgrowth-py,
+max_chars50000. stdout2727..2734 records the exact selection, installed Python
+source path and Lines included487; the parent's author record reports no
+truncation. It is a mechanism reference, not case-specific biological/parity
+acceptance, and does not dispose of or repair the failed ExampleHuman read.
+
+The timeout's captured stderr tail also includes earlier Inspect plate path
+progress and a BigStitcher jar message. That mixed tail is retained evidence,
+not a demonstrated causal stack for knowledge conversion. Do not attribute
+this timeout to Fiji startup, catalogue discovery or the content-size bound
+without further source evidence. Both official30 selections use the existing
+manifest-case conversion owner; MEMB-1/2 still prohibit a case roster, parallel
+registry or selective deadline widening. No new production diagnosis is claimed.
+
+This receipt-only follow-up used the existing WT; no tests, native launches,
+installation, download, timeout change, restart or pending-read replay. Parent
+logs, original failure archives, scientific output and frozen paired installation
+remain untouched. No new disposable build/test/cache directory was created.

docs/validation/knowledge-selected-source-tests.py

@@ -0,0 +1,34 @@
+"""Bounded source checks using the existing read-only ABI environment."""
+
+import importlib.util
+import sys
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[2]
+sys.path[:0] = [
+    str(root),
+    "/home/ts/wt/metaclass-registry-selected-discovery-376-20261001/src",
+]
+installed = Path(
+    "/home/ts/wt/openhcs-generated-inputs-installed-parent-20261001/.venv/lib/python3.12/site-packages/openhcs"
+)
+for name, relative in (
+    ("openhcs.core._tabular_native", "core/_tabular_native.abi3.so"),
+    ("openhcs.processing.backends.cellprofiler._granularity_native",
+     "processing/backends/cellprofiler/_granularity_native.abi3.so"),
+):
+    spec = importlib.util.spec_from_file_location(name, installed / relative)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+
+import pytest
+import metaclass_registry
+
+print("Interpreter:", sys.executable, sys.version, flush=True)
+print("Discovery owner:", metaclass_registry.__file__, flush=True)
+raise SystemExit(pytest.main([
+    "--noconftest", "-o", "addopts=", "-p", "no:cacheprovider",
+    "--basetemp=/home/ts/.cache/agent-scratch/knowledge-selected-source-376-20261001/pytest",
+    *sys.argv[1:],
+]))

docs/validation/selected-discovery-evidence/INTEGRATION-R0-UNCOMPRESSED-SHA256SUMS

@@ -0,0 +1,2 @@
+f5b3dd93c31a327f42927e3988663f934684560ba6e273df3231a94de0899166  integration-openhcs-r0.json
+6713edd45509cf5e862ed9b778c3fb73f589f4d6197774e6332a1b984bc43c51  integration-dependency-r0.json

docs/validation/selected-discovery-evidence/INTEGRATION-SHA256SUMS

@@ -0,0 +1,7 @@
+fbece378e58ffa81e784c832eb49e12768a21c1cc4ebac11696ee413a36c417b  integration-376-commands.sh
+b56a1f6e7357a8ae759227fb2b305128207d28815c771d3e507fb8337f4efec5  integration-focused-source.log
+a17fb99918637752df40c9c13ed3a9d0e1063066e431b3278d6996b9a3ec603c  integration-openhcs-r0.json.gz
+d5ea604cdfb2c7a7aa0146c29643680fb8aae512a8ddcd75dc956687cc7ad13a  integration-openhcs-r0.resources.txt
+1d2724497115a55f624731a4a1dd7fcfb9a8c597420d13a48451ed2a6ac0ebb0  integration-dependency-r0.json.gz
+e5a4e30cf0f41b3b37551048679565e31e376ea495c6efa07bd0dc284094f964  integration-dependency-r0.resources.txt
+be70dafdd273ed3451c0fb856232944ce1c2913d14d637477896115c6bc902d2  INTEGRATION-R0-UNCOMPRESSED-SHA256SUMS

docs/validation/selected-discovery-evidence/POST409-R0-UNCOMPRESSED-SHA256SUMS

@@ -0,0 +1 @@
+c18765b6459437eee54c2d4e6652e449a321888e7e15fc782f80a3f4676f7b48  post409-openhcs-r0.json

docs/validation/selected-discovery-evidence/POST409-SHA256SUMS

@@ -0,0 +1,3 @@
+1882553f43e2c9c9c317da989f0a01e2bc5926fdfc0cc071af21193a33b3f9f3  post409-focused-source.log
+8b6fb413e437868d7d3a8f57d6c95357a82a24513bc08a47ef8aae14751a83aa  post409-openhcs-r0.json.gz
+26f81936ae1fa819759de8fa36ffe8b85b12171dad403e903a7a176333a7c54e  post409-openhcs-r0.resources.txt

docs/validation/selected-discovery-evidence/R0-UNCOMPRESSED-SHA256SUMS

@@ -0,0 +1,5 @@
+5844cdbb7fb97ab9716c705dd78ec84ccff77c1dc8ea3fd4db1576218f946e80  docs/validation/selected-discovery-evidence/openhcs-r0-rejected-5fd.json
+c981ca087763a298ab6c3c8ea229ce96c488b8835889142d6fcac9823ca28b0c  docs/validation/selected-discovery-evidence/openhcs-r0-rejected-24a.json
+4a81ded6871975c8b4fc3e2a336f86b7a82c6755d5f8cf8235b0047af52804d3  docs/validation/selected-discovery-evidence/openhcs-r0-final.json
+d3754f1b852191a1ad33065c018e9bae3b2f46c6fbf9f1ec35d15d9ac243e030  docs/validation/selected-discovery-evidence/dependency-r0.json
+e61b1b6c9b45acf99412c2da06f314230d75cbcb9a416b6cb995ceb372d02175  docs/validation/selected-discovery-evidence/dependency-r0-final.json